2024-11-11 06:39:37,499 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 06:39:37,516 main DEBUG Took 0.014626 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 06:39:37,516 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 06:39:37,517 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 06:39:37,518 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 06:39:37,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,527 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 06:39:37,547 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,550 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,551 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,552 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,554 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,556 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,558 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,559 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,560 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,562 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,563 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,564 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,565 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 06:39:37,567 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,567 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 06:39:37,569 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 06:39:37,571 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 06:39:37,574 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 06:39:37,575 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 06:39:37,577 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 06:39:37,578 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 06:39:37,590 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 06:39:37,592 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 06:39:37,595 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 06:39:37,596 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 06:39:37,597 main DEBUG createAppenders(={Console}) 2024-11-11 06:39:37,598 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 06:39:37,598 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 06:39:37,599 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 06:39:37,600 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 06:39:37,600 main DEBUG OutputStream closed 2024-11-11 06:39:37,600 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 06:39:37,601 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 06:39:37,601 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 06:39:37,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 06:39:37,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 06:39:37,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 06:39:37,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 06:39:37,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 06:39:37,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 06:39:37,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 06:39:37,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 06:39:37,736 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 06:39:37,736 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 06:39:37,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 06:39:37,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 06:39:37,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 06:39:37,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 06:39:37,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 06:39:37,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 06:39:37,740 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 06:39:37,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 06:39:37,744 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 06:39:37,745 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 06:39:37,745 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 06:39:37,746 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T06:39:38,167 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00 2024-11-11 06:39:38,170 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 06:39:38,171 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T06:39:38,178 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-11T06:39:38,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T06:39:38,229 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/cluster_4468b6fc-5992-71c3-8666-e28d197c8ed7, deleteOnExit=true 2024-11-11T06:39:38,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T06:39:38,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/test.cache.data in system properties and HBase conf 2024-11-11T06:39:38,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T06:39:38,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/hadoop.log.dir in system properties and HBase conf 2024-11-11T06:39:38,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T06:39:38,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T06:39:38,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T06:39:38,364 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T06:39:38,494 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T06:39:38,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T06:39:38,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T06:39:38,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T06:39:38,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T06:39:38,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T06:39:38,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T06:39:38,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T06:39:38,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T06:39:38,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T06:39:38,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/nfs.dump.dir in system properties and HBase conf 2024-11-11T06:39:38,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/java.io.tmpdir in system properties and HBase conf 2024-11-11T06:39:38,511 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T06:39:38,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T06:39:38,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T06:39:39,714 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T06:39:39,791 INFO [Time-limited test {}] log.Log(170): Logging initialized @3153ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T06:39:39,885 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T06:39:39,983 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T06:39:40,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T06:39:40,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T06:39:40,023 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T06:39:40,043 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T06:39:40,053 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/hadoop.log.dir/,AVAILABLE} 2024-11-11T06:39:40,054 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T06:39:40,276 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/java.io.tmpdir/jetty-localhost-41147-hadoop-hdfs-3_4_1-tests_jar-_-any-13522202925640180095/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T06:39:40,285 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41147} 2024-11-11T06:39:40,286 INFO [Time-limited test {}] server.Server(415): Started @3649ms 2024-11-11T06:39:40,863 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T06:39:40,873 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T06:39:40,874 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T06:39:40,874 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T06:39:40,874 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T06:39:40,875 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/hadoop.log.dir/,AVAILABLE} 2024-11-11T06:39:40,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T06:39:40,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/java.io.tmpdir/jetty-localhost-34101-hadoop-hdfs-3_4_1-tests_jar-_-any-382815546224609468/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T06:39:40,990 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:34101} 2024-11-11T06:39:40,991 INFO [Time-limited test {}] server.Server(415): Started @4354ms 2024-11-11T06:39:41,038 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T06:39:41,857 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/cluster_4468b6fc-5992-71c3-8666-e28d197c8ed7/data/data1/current/BP-405970163-172.17.0.2-1731307179225/current, will proceed with Du for space computation calculation, 2024-11-11T06:39:41,857 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/cluster_4468b6fc-5992-71c3-8666-e28d197c8ed7/data/data2/current/BP-405970163-172.17.0.2-1731307179225/current, will proceed with Du for space computation calculation, 2024-11-11T06:39:41,882 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T06:39:41,924 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf3a5c2fa34c0f913 with lease ID 0x55727b82b04d6e72: Processing first storage report for DS-781adc9f-4c20-45d8-9261-9698b21dc9a3 from datanode DatanodeRegistration(127.0.0.1:36917, datanodeUuid=cc6721fe-723b-4614-bc5f-d27caf25327e, infoPort=38643, infoSecurePort=0, ipcPort=35405, storageInfo=lv=-57;cid=testClusterID;nsid=1537215564;c=1731307179225) 2024-11-11T06:39:41,925 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf3a5c2fa34c0f913 with lease ID 0x55727b82b04d6e72: from storage DS-781adc9f-4c20-45d8-9261-9698b21dc9a3 node DatanodeRegistration(127.0.0.1:36917, datanodeUuid=cc6721fe-723b-4614-bc5f-d27caf25327e, infoPort=38643, infoSecurePort=0, ipcPort=35405, storageInfo=lv=-57;cid=testClusterID;nsid=1537215564;c=1731307179225), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T06:39:41,926 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf3a5c2fa34c0f913 with lease ID 0x55727b82b04d6e72: Processing first storage report for DS-5e472168-17a9-4b17-83f3-b6b143062931 from datanode DatanodeRegistration(127.0.0.1:36917, datanodeUuid=cc6721fe-723b-4614-bc5f-d27caf25327e, infoPort=38643, infoSecurePort=0, ipcPort=35405, storageInfo=lv=-57;cid=testClusterID;nsid=1537215564;c=1731307179225) 2024-11-11T06:39:41,926 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf3a5c2fa34c0f913 with lease ID 0x55727b82b04d6e72: from storage DS-5e472168-17a9-4b17-83f3-b6b143062931 node DatanodeRegistration(127.0.0.1:36917, datanodeUuid=cc6721fe-723b-4614-bc5f-d27caf25327e, infoPort=38643, infoSecurePort=0, ipcPort=35405, storageInfo=lv=-57;cid=testClusterID;nsid=1537215564;c=1731307179225), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T06:39:41,930 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00 2024-11-11T06:39:41,996 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/cluster_4468b6fc-5992-71c3-8666-e28d197c8ed7/zookeeper_0, clientPort=64841, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/cluster_4468b6fc-5992-71c3-8666-e28d197c8ed7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/cluster_4468b6fc-5992-71c3-8666-e28d197c8ed7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T06:39:42,006 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64841 2024-11-11T06:39:42,019 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T06:39:42,024 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T06:39:42,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741825_1001 (size=7) 2024-11-11T06:39:42,665 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 with version=8 2024-11-11T06:39:42,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/hbase-staging 2024-11-11T06:39:42,751 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T06:39:43,012 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ce2cf0244921:0 server-side Connection retries=45 2024-11-11T06:39:43,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T06:39:43,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T06:39:43,033 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T06:39:43,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T06:39:43,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T06:39:43,203 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T06:39:43,264 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T06:39:43,274 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T06:39:43,278 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T06:39:43,302 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 991 (auto-detected) 2024-11-11T06:39:43,303 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T06:39:43,329 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40845 2024-11-11T06:39:43,348 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40845 connecting to ZooKeeper ensemble=127.0.0.1:64841 2024-11-11T06:39:43,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:408450x0, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T06:39:43,449 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40845-0x10129e8aa610000 connected 2024-11-11T06:39:43,527 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T06:39:43,529 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T06:39:43,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T06:39:43,546 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416, hbase.cluster.distributed=false 2024-11-11T06:39:43,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T06:39:43,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40845 2024-11-11T06:39:43,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40845 2024-11-11T06:39:43,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40845 2024-11-11T06:39:43,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40845 2024-11-11T06:39:43,585 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40845 2024-11-11T06:39:43,710 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ce2cf0244921:0 server-side Connection retries=45 2024-11-11T06:39:43,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T06:39:43,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T06:39:43,712 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T06:39:43,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T06:39:43,712 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T06:39:43,716 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T06:39:43,718 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T06:39:43,719 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35807 2024-11-11T06:39:43,721 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35807 connecting to ZooKeeper ensemble=127.0.0.1:64841 2024-11-11T06:39:43,723 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T06:39:43,726 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T06:39:43,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358070x0, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T06:39:43,766 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:358070x0, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T06:39:43,767 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35807-0x10129e8aa610001 connected 2024-11-11T06:39:43,770 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T06:39:43,780 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T06:39:43,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T06:39:43,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T06:39:43,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-11T06:39:43,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35807 2024-11-11T06:39:43,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35807 2024-11-11T06:39:43,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-11T06:39:43,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35807 2024-11-11T06:39:43,823 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ce2cf0244921:40845 2024-11-11T06:39:43,824 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ce2cf0244921,40845,1731307182847 2024-11-11T06:39:43,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T06:39:43,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T06:39:43,845 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ce2cf0244921,40845,1731307182847 2024-11-11T06:39:43,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T06:39:43,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:43,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:43,876 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T06:39:43,878 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ce2cf0244921,40845,1731307182847 from backup master directory 2024-11-11T06:39:43,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ce2cf0244921,40845,1731307182847 2024-11-11T06:39:43,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T06:39:43,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T06:39:43,892 WARN [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T06:39:43,893 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ce2cf0244921,40845,1731307182847 2024-11-11T06:39:43,895 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T06:39:43,897 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T06:39:43,962 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/hbase.id] with ID: 16654907-934f-4fd9-82c1-8c338d43dded 2024-11-11T06:39:43,962 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/.tmp/hbase.id 2024-11-11T06:39:43,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741826_1002 (size=42) 2024-11-11T06:39:44,375 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/.tmp/hbase.id]:[hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/hbase.id] 2024-11-11T06:39:44,426 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T06:39:44,432 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T06:39:44,450 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-11T06:39:44,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:44,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:44,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741827_1003 (size=196) 2024-11-11T06:39:44,954 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:39:44,957 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T06:39:44,971 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:44,975 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T06:39:45,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741828_1004 (size=1189) 2024-11-11T06:39:45,425 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store 2024-11-11T06:39:45,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741829_1005 (size=34) 2024-11-11T06:39:45,854 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T06:39:45,856 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:39:45,857 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T06:39:45,858 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T06:39:45,858 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T06:39:45,859 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T06:39:45,859 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T06:39:45,859 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T06:39:45,861 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731307185857Disabling compacts and flushes for region at 1731307185857Disabling writes for close at 1731307185859 (+2 ms)Writing region close event to WAL at 1731307185859Closed at 1731307185859 2024-11-11T06:39:45,863 WARN [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/.initializing 2024-11-11T06:39:45,863 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/WALs/ce2cf0244921,40845,1731307182847 2024-11-11T06:39:45,871 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T06:39:45,886 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ce2cf0244921%2C40845%2C1731307182847, suffix=, logDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/WALs/ce2cf0244921,40845,1731307182847, archiveDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/oldWALs, maxLogs=10 2024-11-11T06:39:45,905 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/WALs/ce2cf0244921,40845,1731307182847/ce2cf0244921%2C40845%2C1731307182847.1731307185889, exclude list is [], retry=0 2024-11-11T06:39:45,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36917,DS-781adc9f-4c20-45d8-9261-9698b21dc9a3,DISK] 2024-11-11T06:39:45,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T06:39:45,957 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/WALs/ce2cf0244921,40845,1731307182847/ce2cf0244921%2C40845%2C1731307182847.1731307185889 2024-11-11T06:39:45,958 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38643:38643)] 2024-11-11T06:39:45,958 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:39:45,959 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:39:45,962 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:45,963 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T06:39:46,032 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:46,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T06:39:46,039 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:39:46,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T06:39:46,045 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:39:46,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T06:39:46,050 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:39:46,052 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,055 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,056 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,063 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,064 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,069 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T06:39:46,073 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T06:39:46,079 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T06:39:46,080 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65798771, jitterRate=-0.019521906971931458}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T06:39:46,088 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731307185976Initializing all the Stores at 1731307185978 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731307185979 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307185980 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307185980Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307185980Cleaning up temporary data from old regions at 1731307186064 (+84 ms)Region opened successfully at 1731307186088 (+24 ms) 2024-11-11T06:39:46,090 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T06:39:46,126 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@353f3d52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ce2cf0244921/172.17.0.2:0 2024-11-11T06:39:46,159 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T06:39:46,172 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T06:39:46,172 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T06:39:46,176 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T06:39:46,177 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T06:39:46,183 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-11T06:39:46,183 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T06:39:46,210 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T06:39:46,222 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T06:39:46,274 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T06:39:46,276 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T06:39:46,278 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T06:39:46,282 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T06:39:46,285 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T06:39:46,289 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T06:39:46,299 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T06:39:46,301 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T06:39:46,307 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T06:39:46,331 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T06:39:46,341 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T06:39:46,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T06:39:46,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T06:39:46,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:46,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:46,353 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ce2cf0244921,40845,1731307182847, sessionid=0x10129e8aa610000, setting cluster-up flag (Was=false) 2024-11-11T06:39:46,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:46,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:46,407 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T06:39:46,409 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ce2cf0244921,40845,1731307182847 2024-11-11T06:39:46,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:46,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:46,457 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T06:39:46,460 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ce2cf0244921,40845,1731307182847 2024-11-11T06:39:46,467 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T06:39:46,515 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(746): ClusterId : 16654907-934f-4fd9-82c1-8c338d43dded 2024-11-11T06:39:46,518 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T06:39:46,535 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T06:39:46,535 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T06:39:46,542 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T06:39:46,543 DEBUG [RS:0;ce2cf0244921:35807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f063daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ce2cf0244921/172.17.0.2:0 2024-11-11T06:39:46,551 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T06:39:46,559 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ce2cf0244921:35807 2024-11-11T06:39:46,563 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T06:39:46,563 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T06:39:46,563 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T06:39:46,563 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T06:39:46,565 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(2659): reportForDuty to master=ce2cf0244921,40845,1731307182847 with port=35807, startcode=1731307183669 2024-11-11T06:39:46,571 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T06:39:46,580 DEBUG [RS:0;ce2cf0244921:35807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T06:39:46,578 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ce2cf0244921,40845,1731307182847 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T06:39:46,587 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ce2cf0244921:0, corePoolSize=5, maxPoolSize=5 2024-11-11T06:39:46,588 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ce2cf0244921:0, corePoolSize=5, maxPoolSize=5 2024-11-11T06:39:46,588 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ce2cf0244921:0, corePoolSize=5, maxPoolSize=5 2024-11-11T06:39:46,588 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ce2cf0244921:0, corePoolSize=5, maxPoolSize=5 2024-11-11T06:39:46,588 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ce2cf0244921:0, corePoolSize=10, maxPoolSize=10 2024-11-11T06:39:46,588 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,589 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ce2cf0244921:0, corePoolSize=2, maxPoolSize=2 2024-11-11T06:39:46,589 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,593 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T06:39:46,594 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T06:39:46,600 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,601 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T06:39:46,614 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731307216613 2024-11-11T06:39:46,616 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T06:39:46,617 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T06:39:46,621 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T06:39:46,622 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T06:39:46,623 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T06:39:46,624 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T06:39:46,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741831_1007 (size=1321) 2024-11-11T06:39:46,631 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,636 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T06:39:46,637 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 2024-11-11T06:39:46,639 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T06:39:46,641 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T06:39:46,641 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38879, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T06:39:46,643 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T06:39:46,650 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40845 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:46,651 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T06:39:46,651 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T06:39:46,654 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.large.0-1731307186653,5,FailOnTimeoutGroup] 2024-11-11T06:39:46,659 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.small.0-1731307186655,5,FailOnTimeoutGroup] 2024-11-11T06:39:46,659 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,659 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T06:39:46,661 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,661 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741832_1008 (size=32) 2024-11-11T06:39:46,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:39:46,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T06:39:46,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T06:39:46,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:46,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T06:39:46,683 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-11T06:39:46,683 WARN [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T06:39:46,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T06:39:46,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:46,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T06:39:46,688 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T06:39:46,688 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:46,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T06:39:46,693 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T06:39:46,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:46,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:46,695 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T06:39:46,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740 2024-11-11T06:39:46,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740 2024-11-11T06:39:46,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T06:39:46,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T06:39:46,705 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:39:46,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T06:39:46,713 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T06:39:46,713 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62612406, jitterRate=-0.06700244545936584}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:39:46,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731307186668Initializing all the Stores at 1731307186671 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731307186671Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731307186674 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307186675 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731307186675Cleaning up temporary data from old regions at 1731307186704 (+29 ms)Region opened successfully at 1731307186717 (+13 ms) 2024-11-11T06:39:46,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T06:39:46,718 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T06:39:46,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T06:39:46,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T06:39:46,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T06:39:46,719 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T06:39:46,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731307186717Disabling compacts and flushes for region at 1731307186717Disabling writes for close at 1731307186718 (+1 ms)Writing region close event to WAL at 1731307186719 (+1 ms)Closed at 1731307186719 2024-11-11T06:39:46,722 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T06:39:46,723 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T06:39:46,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T06:39:46,739 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T06:39:46,742 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T06:39:46,785 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(2659): reportForDuty to master=ce2cf0244921,40845,1731307182847 with port=35807, startcode=1731307183669 2024-11-11T06:39:46,788 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40845 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ce2cf0244921,35807,1731307183669 2024-11-11T06:39:46,792 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40845 {}] master.ServerManager(517): Registering regionserver=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:46,803 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 2024-11-11T06:39:46,803 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36813 2024-11-11T06:39:46,803 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T06:39:46,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T06:39:46,833 DEBUG [RS:0;ce2cf0244921:35807 {}] zookeeper.ZKUtil(111): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ce2cf0244921,35807,1731307183669 2024-11-11T06:39:46,833 WARN [RS:0;ce2cf0244921:35807 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T06:39:46,834 INFO [RS:0;ce2cf0244921:35807 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T06:39:46,834 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/WALs/ce2cf0244921,35807,1731307183669 2024-11-11T06:39:46,837 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ce2cf0244921,35807,1731307183669] 2024-11-11T06:39:46,860 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T06:39:46,874 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T06:39:46,879 INFO [RS:0;ce2cf0244921:35807 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T06:39:46,879 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,880 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T06:39:46,885 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T06:39:46,886 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,886 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,886 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,886 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ce2cf0244921:0, corePoolSize=2, maxPoolSize=2 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,887 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T06:39:46,888 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ce2cf0244921:0, corePoolSize=3, maxPoolSize=3 2024-11-11T06:39:46,888 DEBUG [RS:0;ce2cf0244921:35807 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0, corePoolSize=3, maxPoolSize=3 2024-11-11T06:39:46,888 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,889 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,889 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,889 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,889 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,889 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,35807,1731307183669-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T06:39:46,892 WARN [ce2cf0244921:40845 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T06:39:46,906 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T06:39:46,907 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,35807,1731307183669-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,908 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,908 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.Replication(171): ce2cf0244921,35807,1731307183669 started 2024-11-11T06:39:46,923 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:46,923 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1482): Serving as ce2cf0244921,35807,1731307183669, RpcServer on ce2cf0244921/172.17.0.2:35807, sessionid=0x10129e8aa610001 2024-11-11T06:39:46,924 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T06:39:46,924 DEBUG [RS:0;ce2cf0244921:35807 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ce2cf0244921,35807,1731307183669 2024-11-11T06:39:46,924 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ce2cf0244921,35807,1731307183669' 2024-11-11T06:39:46,925 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T06:39:46,925 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T06:39:46,926 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T06:39:46,926 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T06:39:46,926 DEBUG [RS:0;ce2cf0244921:35807 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ce2cf0244921,35807,1731307183669 2024-11-11T06:39:46,926 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ce2cf0244921,35807,1731307183669' 2024-11-11T06:39:46,926 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T06:39:46,927 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T06:39:46,927 DEBUG [RS:0;ce2cf0244921:35807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T06:39:46,928 INFO [RS:0;ce2cf0244921:35807 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T06:39:46,928 INFO [RS:0;ce2cf0244921:35807 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T06:39:47,034 INFO [RS:0;ce2cf0244921:35807 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T06:39:47,039 INFO [RS:0;ce2cf0244921:35807 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ce2cf0244921%2C35807%2C1731307183669, suffix=, logDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/WALs/ce2cf0244921,35807,1731307183669, archiveDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/oldWALs, maxLogs=32 2024-11-11T06:39:47,053 DEBUG [RS:0;ce2cf0244921:35807 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/WALs/ce2cf0244921,35807,1731307183669/ce2cf0244921%2C35807%2C1731307183669.1731307187041, exclude list is [], retry=0 2024-11-11T06:39:47,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36917,DS-781adc9f-4c20-45d8-9261-9698b21dc9a3,DISK] 2024-11-11T06:39:47,061 INFO [RS:0;ce2cf0244921:35807 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/WALs/ce2cf0244921,35807,1731307183669/ce2cf0244921%2C35807%2C1731307183669.1731307187041 2024-11-11T06:39:47,061 DEBUG [RS:0;ce2cf0244921:35807 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38643:38643)] 2024-11-11T06:39:47,145 DEBUG [ce2cf0244921:40845 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T06:39:47,159 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:47,164 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ce2cf0244921,35807,1731307183669, state=OPENING 2024-11-11T06:39:47,207 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T06:39:47,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:47,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:39:47,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T06:39:47,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T06:39:47,219 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T06:39:47,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:39:47,405 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T06:39:47,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43411, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T06:39:47,418 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T06:39:47,418 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T06:39:47,419 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T06:39:47,422 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ce2cf0244921%2C35807%2C1731307183669.meta, suffix=.meta, logDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/WALs/ce2cf0244921,35807,1731307183669, archiveDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/oldWALs, maxLogs=32 2024-11-11T06:39:47,435 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/WALs/ce2cf0244921,35807,1731307183669/ce2cf0244921%2C35807%2C1731307183669.meta.1731307187424.meta, exclude list is [], retry=0 2024-11-11T06:39:47,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36917,DS-781adc9f-4c20-45d8-9261-9698b21dc9a3,DISK] 2024-11-11T06:39:47,442 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/WALs/ce2cf0244921,35807,1731307183669/ce2cf0244921%2C35807%2C1731307183669.meta.1731307187424.meta 2024-11-11T06:39:47,442 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38643:38643)] 2024-11-11T06:39:47,442 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:39:47,444 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T06:39:47,446 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T06:39:47,450 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T06:39:47,454 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T06:39:47,454 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:39:47,455 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T06:39:47,455 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T06:39:47,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T06:39:47,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T06:39:47,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:47,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:47,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T06:39:47,462 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T06:39:47,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:47,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:47,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T06:39:47,464 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T06:39:47,464 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:47,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:47,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T06:39:47,466 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T06:39:47,466 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:47,467 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T06:39:47,467 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T06:39:47,469 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740 2024-11-11T06:39:47,472 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740 2024-11-11T06:39:47,475 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T06:39:47,475 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T06:39:47,476 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:39:47,478 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T06:39:47,480 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70468325, jitterRate=0.05005986988544464}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:39:47,480 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T06:39:47,482 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731307187455Writing region info on filesystem at 1731307187455Initializing all the Stores at 1731307187457 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731307187457Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731307187457Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307187457Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731307187457Cleaning up temporary data from old regions at 1731307187475 (+18 ms)Running coprocessor post-open hooks at 1731307187480 (+5 ms)Region opened successfully at 1731307187482 (+2 ms) 2024-11-11T06:39:47,488 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731307187396 2024-11-11T06:39:47,498 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T06:39:47,499 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T06:39:47,500 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:47,503 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ce2cf0244921,35807,1731307183669, state=OPEN 2024-11-11T06:39:47,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T06:39:47,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T06:39:47,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T06:39:47,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T06:39:47,596 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:47,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T06:39:47,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ce2cf0244921,35807,1731307183669 in 374 msec 2024-11-11T06:39:47,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T06:39:47,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 875 msec 2024-11-11T06:39:47,612 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T06:39:47,612 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T06:39:47,630 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:47,631 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:47,648 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:47,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41751, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:47,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1640 sec 2024-11-11T06:39:47,667 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731307187667, completionTime=-1 2024-11-11T06:39:47,670 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T06:39:47,670 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T06:39:47,719 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T06:39:47,719 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731307247719 2024-11-11T06:39:47,719 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731307307719 2024-11-11T06:39:47,719 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 49 msec 2024-11-11T06:39:47,721 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,40845,1731307182847-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:47,721 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,40845,1731307182847-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:47,721 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,40845,1731307182847-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:47,723 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ce2cf0244921:40845, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:47,723 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:47,723 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:47,730 DEBUG [master/ce2cf0244921:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T06:39:47,749 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.855sec 2024-11-11T06:39:47,749 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T06:39:47,750 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T06:39:47,751 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T06:39:47,752 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T06:39:47,752 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T06:39:47,752 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,40845,1731307182847-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T06:39:47,753 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,40845,1731307182847-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T06:39:47,760 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T06:39:47,761 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T06:39:47,761 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,40845,1731307182847-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T06:39:47,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:47,833 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T06:39:47,833 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T06:39:47,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:47,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:47,844 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b40305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:47,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:47,853 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:47,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:47,856 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:47,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:47,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ce2cf0244921,40845,1731307182847 2024-11-11T06:39:47,894 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=319, ProcessCount=11, AvailableMemoryMB=12158 2024-11-11T06:39:47,901 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T06:39:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,40845,1731307182847 2024-11-11T06:39:47,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3289bc03 2024-11-11T06:39:47,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T06:39:47,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T06:39:47,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:39:47,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:39:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T06:39:47,930 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T06:39:47,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T06:39:47,932 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:47,935 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T06:39:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T06:39:47,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741835_1011 (size=959) 2024-11-11T06:39:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T06:39:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T06:39:48,351 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 2024-11-11T06:39:48,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741836_1012 (size=53) 2024-11-11T06:39:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T06:39:48,763 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:39:48,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1b6d451b960d07000ebc207f625528a7, disabling compactions & flushes 2024-11-11T06:39:48,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:48,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:48,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. after waiting 0 ms 2024-11-11T06:39:48,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:48,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:48,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1b6d451b960d07000ebc207f625528a7: Waiting for close lock at 1731307188764Disabling compacts and flushes for region at 1731307188764Disabling writes for close at 1731307188764Writing region close event to WAL at 1731307188764Closed at 1731307188764 2024-11-11T06:39:48,768 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T06:39:48,774 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731307188768"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731307188768"}]},"ts":"1731307188768"} 2024-11-11T06:39:48,779 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T06:39:48,782 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T06:39:48,785 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307188782"}]},"ts":"1731307188782"} 2024-11-11T06:39:48,790 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T06:39:48,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, ASSIGN}] 2024-11-11T06:39:48,796 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, ASSIGN 2024-11-11T06:39:48,798 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, ASSIGN; state=OFFLINE, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=false 2024-11-11T06:39:48,950 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b6d451b960d07000ebc207f625528a7, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:48,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, ASSIGN because future has completed 2024-11-11T06:39:48,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:39:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T06:39:49,120 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:49,120 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:39:49,121 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,121 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:39:49,121 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,121 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,124 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,127 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b6d451b960d07000ebc207f625528a7 columnFamilyName A 2024-11-11T06:39:49,127 DEBUG [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:49,128 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] regionserver.HStore(327): Store=1b6d451b960d07000ebc207f625528a7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:39:49,128 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,131 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b6d451b960d07000ebc207f625528a7 columnFamilyName B 2024-11-11T06:39:49,131 DEBUG [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:49,132 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] regionserver.HStore(327): Store=1b6d451b960d07000ebc207f625528a7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:39:49,132 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,135 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b6d451b960d07000ebc207f625528a7 columnFamilyName C 2024-11-11T06:39:49,135 DEBUG [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:39:49,137 INFO [StoreOpener-1b6d451b960d07000ebc207f625528a7-1 {}] regionserver.HStore(327): Store=1b6d451b960d07000ebc207f625528a7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:39:49,137 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,139 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,140 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,143 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,143 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,144 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:39:49,147 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,152 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T06:39:49,153 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1b6d451b960d07000ebc207f625528a7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62713006, jitterRate=-0.06550338864326477}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:39:49,154 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:49,155 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1b6d451b960d07000ebc207f625528a7: Running coprocessor pre-open hook at 1731307189121Writing region info on filesystem at 1731307189121Initializing all the Stores at 1731307189123 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307189123Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307189124 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307189124Cleaning up temporary data from old regions at 1731307189143 (+19 ms)Running coprocessor post-open hooks at 1731307189154 (+11 ms)Region opened successfully at 1731307189155 (+1 ms) 2024-11-11T06:39:49,158 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., pid=6, masterSystemTime=1731307189112 2024-11-11T06:39:49,163 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:49,163 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:49,165 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b6d451b960d07000ebc207f625528a7, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:49,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:39:49,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T06:39:49,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 in 217 msec 2024-11-11T06:39:49,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T06:39:49,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, ASSIGN in 394 msec 2024-11-11T06:39:49,194 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T06:39:49,194 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307189194"}]},"ts":"1731307189194"} 2024-11-11T06:39:49,198 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T06:39:49,200 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T06:39:49,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2790 sec 2024-11-11T06:39:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T06:39:50,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:39:50,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c32821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,111 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f534466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297ba807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,116 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T06:39:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,40845,1731307182847 2024-11-11T06:39:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52a4d08 2024-11-11T06:39:50,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T06:39:50,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T06:39:50,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd90ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,136 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ae7626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,140 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d2795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,144 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,148 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5a580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba02c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,152 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf2c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,156 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113d617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,160 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d79f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51273011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2754d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,166 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@211dc17b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,169 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b420fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ddbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,172 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f759963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,175 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369fd76d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a48b566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,178 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b0da03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc04dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,184 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b54236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,187 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:39:50,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:39:50,190 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:39:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:39:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:39:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d424882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:39:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:39:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:39:50,193 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:39:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293d4ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:39:50,197 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,197 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,197 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,197 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,198 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,200 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,200 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,201 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,201 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,207 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,208 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:39:50,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:39:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:39:50,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T06:39:50,222 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:39:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T06:39:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:39:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:39:50,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:39:50,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:39:50,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:39:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T06:39:50,339 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:39:50,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:39:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:50,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:50,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:50,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:50,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:50,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/0f30091c3b1a4c11a61701a5400e4af2 is 50, key is test_row_0/A:col10/1731307190290/Put/seqid=0 2024-11-11T06:39:50,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741837_1013 (size=12001) 2024-11-11T06:39:50,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/0f30091c3b1a4c11a61701a5400e4af2 2024-11-11T06:39:50,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307250512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307250512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307250521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307250521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307250522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T06:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:50,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:50,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:50,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:50,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2eeabead1ac346a481255bb7f3f96525 is 50, key is test_row_0/B:col10/1731307190290/Put/seqid=0 2024-11-11T06:39:50,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741838_1014 (size=12001) 2024-11-11T06:39:50,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2eeabead1ac346a481255bb7f3f96525 2024-11-11T06:39:50,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307250641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307250642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307250641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307250650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307250651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/d9624ffb013c4aa2aa812e3868e78619 is 50, key is test_row_0/C:col10/1731307190290/Put/seqid=0 2024-11-11T06:39:50,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741839_1015 (size=12001) 2024-11-11T06:39:50,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/d9624ffb013c4aa2aa812e3868e78619 2024-11-11T06:39:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/0f30091c3b1a4c11a61701a5400e4af2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/0f30091c3b1a4c11a61701a5400e4af2 2024-11-11T06:39:50,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:50,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/0f30091c3b1a4c11a61701a5400e4af2, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T06:39:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:50,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2eeabead1ac346a481255bb7f3f96525 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2eeabead1ac346a481255bb7f3f96525 2024-11-11T06:39:50,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2eeabead1ac346a481255bb7f3f96525, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T06:39:50,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/d9624ffb013c4aa2aa812e3868e78619 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/d9624ffb013c4aa2aa812e3868e78619 2024-11-11T06:39:50,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/d9624ffb013c4aa2aa812e3868e78619, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T06:39:50,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 1b6d451b960d07000ebc207f625528a7 in 460ms, sequenceid=14, compaction requested=false 2024-11-11T06:39:50,809 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T06:39:50,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T06:39:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:50,879 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:39:50,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T06:39:50,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dd9f12c23d954818b77994753e18d743 is 50, key is test_row_0/A:col10/1731307190875/Put/seqid=0 2024-11-11T06:39:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307250896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:50,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:50,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:50,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307250899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741840_1016 (size=12001) 2024-11-11T06:39:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dd9f12c23d954818b77994753e18d743 2024-11-11T06:39:50,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307250901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307250904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307250905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:50,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:50,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/18340bc4030543d180c54e059374b882 is 50, key is test_row_0/B:col10/1731307190875/Put/seqid=0 2024-11-11T06:39:50,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741841_1017 (size=12001) 2024-11-11T06:39:51,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307251009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307251030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307251042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307251042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307251043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:51,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:51,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307251229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307251251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307251250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307251251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307251260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T06:39:51,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/18340bc4030543d180c54e059374b882 2024-11-11T06:39:51,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:51,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/abe7459cb5f24b81ac73bd8f7761bb0e is 50, key is test_row_0/C:col10/1731307190875/Put/seqid=0 2024-11-11T06:39:51,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741842_1018 (size=12001) 2024-11-11T06:39:51,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:51,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:51,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307251540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307251560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307251570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307251570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307251580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:51,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:51,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:51,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:51,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/abe7459cb5f24b81ac73bd8f7761bb0e 2024-11-11T06:39:51,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:51,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dd9f12c23d954818b77994753e18d743 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dd9f12c23d954818b77994753e18d743 2024-11-11T06:39:51,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dd9f12c23d954818b77994753e18d743, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T06:39:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/18340bc4030543d180c54e059374b882 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/18340bc4030543d180c54e059374b882 2024-11-11T06:39:51,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/18340bc4030543d180c54e059374b882, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T06:39:51,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/abe7459cb5f24b81ac73bd8f7761bb0e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/abe7459cb5f24b81ac73bd8f7761bb0e 2024-11-11T06:39:51,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/abe7459cb5f24b81ac73bd8f7761bb0e, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T06:39:51,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1b6d451b960d07000ebc207f625528a7 in 1051ms, sequenceid=41, compaction requested=false 2024-11-11T06:39:51,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:52,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T06:39:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:52,012 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T06:39:52,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/f26978258a0240a091f42cf7478dc388 is 50, key is test_row_0/A:col10/1731307190895/Put/seqid=0 2024-11-11T06:39:52,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741843_1019 (size=9657) 2024-11-11T06:39:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:52,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307252150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307252156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307252156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307252160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307252163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307252271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307252271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307252272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307252288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307252288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T06:39:52,461 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/f26978258a0240a091f42cf7478dc388 2024-11-11T06:39:52,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307252479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/987a4a0989ce4f2184437d30c308a146 is 50, key is test_row_0/B:col10/1731307190895/Put/seqid=0 2024-11-11T06:39:52,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307252489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307252490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307252500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307252502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741844_1020 (size=9657) 2024-11-11T06:39:52,522 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/987a4a0989ce4f2184437d30c308a146 2024-11-11T06:39:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/f3aa4238553b4c2bba6c2ba8858a6564 is 50, key is test_row_0/C:col10/1731307190895/Put/seqid=0 2024-11-11T06:39:52,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741845_1021 (size=9657) 2024-11-11T06:39:52,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307252800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307252800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307252801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307252810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307252810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:52,966 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T06:39:52,983 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/f3aa4238553b4c2bba6c2ba8858a6564 2024-11-11T06:39:53,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/f26978258a0240a091f42cf7478dc388 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f26978258a0240a091f42cf7478dc388 2024-11-11T06:39:53,018 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f26978258a0240a091f42cf7478dc388, entries=100, sequenceid=50, filesize=9.4 K 2024-11-11T06:39:53,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/987a4a0989ce4f2184437d30c308a146 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/987a4a0989ce4f2184437d30c308a146 2024-11-11T06:39:53,044 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/987a4a0989ce4f2184437d30c308a146, entries=100, sequenceid=50, filesize=9.4 K 2024-11-11T06:39:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/f3aa4238553b4c2bba6c2ba8858a6564 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f3aa4238553b4c2bba6c2ba8858a6564 2024-11-11T06:39:53,051 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T06:39:53,063 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f3aa4238553b4c2bba6c2ba8858a6564, entries=100, sequenceid=50, filesize=9.4 K 2024-11-11T06:39:53,066 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 1b6d451b960d07000ebc207f625528a7 in 1054ms, sequenceid=50, compaction requested=true 2024-11-11T06:39:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T06:39:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T06:39:53,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T06:39:53,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8390 sec 2024-11-11T06:39:53,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.8640 sec 2024-11-11T06:39:53,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T06:39:53,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T06:39:53,263 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T06:39:53,263 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T06:39:53,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T06:39:53,264 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T06:39:53,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T06:39:53,266 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T06:39:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:53,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:39:53,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-11T06:39:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307253330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307253330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307253334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307253339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307253334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/6a6fa7779d7741f8960f5ca4f2e8c8b8 is 50, key is test_row_0/A:col10/1731307193321/Put/seqid=0 2024-11-11T06:39:53,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741846_1022 (size=12001) 2024-11-11T06:39:53,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/6a6fa7779d7741f8960f5ca4f2e8c8b8 2024-11-11T06:39:53,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/7d499d4a8f40440e85280a88a09901c2 is 50, key is test_row_0/B:col10/1731307193321/Put/seqid=0 2024-11-11T06:39:53,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307253460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307253460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307253460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307253472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741847_1023 (size=12001) 2024-11-11T06:39:53,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/7d499d4a8f40440e85280a88a09901c2 2024-11-11T06:39:53,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b9533b08d4eb45299d891462aece5678 is 50, key is test_row_0/C:col10/1731307193321/Put/seqid=0 2024-11-11T06:39:53,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741848_1024 (size=12001) 2024-11-11T06:39:53,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307253680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307253680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307253680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307253680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:53,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b9533b08d4eb45299d891462aece5678 2024-11-11T06:39:53,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/6a6fa7779d7741f8960f5ca4f2e8c8b8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/6a6fa7779d7741f8960f5ca4f2e8c8b8 2024-11-11T06:39:53,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307253989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307253990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307253999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307253989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/6a6fa7779d7741f8960f5ca4f2e8c8b8, entries=150, sequenceid=80, filesize=11.7 K 2024-11-11T06:39:54,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/7d499d4a8f40440e85280a88a09901c2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7d499d4a8f40440e85280a88a09901c2 2024-11-11T06:39:54,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7d499d4a8f40440e85280a88a09901c2, entries=150, sequenceid=80, filesize=11.7 K 2024-11-11T06:39:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b9533b08d4eb45299d891462aece5678 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b9533b08d4eb45299d891462aece5678 2024-11-11T06:39:54,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b9533b08d4eb45299d891462aece5678, entries=150, sequenceid=80, filesize=11.7 K 2024-11-11T06:39:54,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 1b6d451b960d07000ebc207f625528a7 in 740ms, sequenceid=80, compaction requested=true 2024-11-11T06:39:54,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:39:54,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:39:54,073 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:39:54,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:39:54,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:39:54,073 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:39:54,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:39:54,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:39:54,082 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:39:54,084 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:39:54,085 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,085 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2eeabead1ac346a481255bb7f3f96525, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/18340bc4030543d180c54e059374b882, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/987a4a0989ce4f2184437d30c308a146, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7d499d4a8f40440e85280a88a09901c2] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=44.6 K 2024-11-11T06:39:54,086 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:39:54,087 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:39:54,087 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,088 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/0f30091c3b1a4c11a61701a5400e4af2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dd9f12c23d954818b77994753e18d743, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f26978258a0240a091f42cf7478dc388, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/6a6fa7779d7741f8960f5ca4f2e8c8b8] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=44.6 K 2024-11-11T06:39:54,089 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eeabead1ac346a481255bb7f3f96525, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731307190290 2024-11-11T06:39:54,090 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 18340bc4030543d180c54e059374b882, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731307190514 2024-11-11T06:39:54,090 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f30091c3b1a4c11a61701a5400e4af2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731307190290 2024-11-11T06:39:54,091 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd9f12c23d954818b77994753e18d743, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731307190514 2024-11-11T06:39:54,092 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting f26978258a0240a091f42cf7478dc388, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307190895 2024-11-11T06:39:54,092 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 987a4a0989ce4f2184437d30c308a146, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307190895 2024-11-11T06:39:54,093 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d499d4a8f40440e85280a88a09901c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731307192154 2024-11-11T06:39:54,093 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a6fa7779d7741f8960f5ca4f2e8c8b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731307192154 2024-11-11T06:39:54,153 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:39:54,154 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/c5a8a83d19a14a0eb0fd7cdc399608be is 50, key is test_row_0/A:col10/1731307193321/Put/seqid=0 2024-11-11T06:39:54,159 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:39:54,160 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/44f59547ef3349bdbfc5a4fbccb76bd4 is 50, key is test_row_0/B:col10/1731307193321/Put/seqid=0 2024-11-11T06:39:54,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741849_1025 (size=12139) 2024-11-11T06:39:54,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741850_1026 (size=12139) 2024-11-11T06:39:54,209 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/44f59547ef3349bdbfc5a4fbccb76bd4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/44f59547ef3349bdbfc5a4fbccb76bd4 2024-11-11T06:39:54,239 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into 44f59547ef3349bdbfc5a4fbccb76bd4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:39:54,239 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:54,242 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=12, startTime=1731307194073; duration=0sec 2024-11-11T06:39:54,243 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:39:54,243 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:39:54,243 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:39:54,247 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:39:54,247 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:39:54,247 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,248 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/d9624ffb013c4aa2aa812e3868e78619, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/abe7459cb5f24b81ac73bd8f7761bb0e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f3aa4238553b4c2bba6c2ba8858a6564, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b9533b08d4eb45299d891462aece5678] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=44.6 K 2024-11-11T06:39:54,249 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting d9624ffb013c4aa2aa812e3868e78619, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731307190290 2024-11-11T06:39:54,271 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting abe7459cb5f24b81ac73bd8f7761bb0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731307190514 2024-11-11T06:39:54,272 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting f3aa4238553b4c2bba6c2ba8858a6564, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307190895 2024-11-11T06:39:54,275 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b9533b08d4eb45299d891462aece5678, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731307192154 2024-11-11T06:39:54,326 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#14 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:39:54,334 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/24f76114a5e2470a883910dc4cdeb250 is 50, key is test_row_0/C:col10/1731307193321/Put/seqid=0 2024-11-11T06:39:54,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741851_1027 (size=12139) 2024-11-11T06:39:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T06:39:54,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:39:54,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:39:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T06:39:54,396 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/24f76114a5e2470a883910dc4cdeb250 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/24f76114a5e2470a883910dc4cdeb250 2024-11-11T06:39:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T06:39:54,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:39:54,398 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:39:54,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:39:54,409 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into 24f76114a5e2470a883910dc4cdeb250(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:39:54,409 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:54,409 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=12, startTime=1731307194073; duration=0sec 2024-11-11T06:39:54,410 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:39:54,410 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:39:54,422 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:39:54,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:39:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:54,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/3e7533b40f9c4641b8890e18caa62d66 is 50, key is test_row_0/A:col10/1731307194420/Put/seqid=0 2024-11-11T06:39:54,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741852_1028 (size=14341) 2024-11-11T06:39:54,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/3e7533b40f9c4641b8890e18caa62d66 2024-11-11T06:39:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T06:39:54,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/9bcc50bb59264fbeaa787bf4dd2fdc03 is 50, key is test_row_0/B:col10/1731307194420/Put/seqid=0 2024-11-11T06:39:54,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T06:39:54,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:54,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741853_1029 (size=12001) 2024-11-11T06:39:54,593 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/c5a8a83d19a14a0eb0fd7cdc399608be as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c5a8a83d19a14a0eb0fd7cdc399608be 2024-11-11T06:39:54,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307254577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307254580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307254586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,606 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into c5a8a83d19a14a0eb0fd7cdc399608be(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:39:54,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,606 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:54,606 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=12, startTime=1731307194066; duration=0sec 2024-11-11T06:39:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307254587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,606 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:39:54,606 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:39:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307254590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T06:39:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307254710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307254710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307254710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307254710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T06:39:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307254719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T06:39:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:54,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:54,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:54,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307254919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307254920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307254920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307254919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307254939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:54,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/9bcc50bb59264fbeaa787bf4dd2fdc03 2024-11-11T06:39:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T06:39:55,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/2f9e8698a197481992ef6d8684e6ec0e is 50, key is test_row_0/C:col10/1731307194420/Put/seqid=0 2024-11-11T06:39:55,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T06:39:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:55,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741854_1030 (size=12001) 2024-11-11T06:39:55,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T06:39:55,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:55,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307255230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307255230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307255230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307255239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307255259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T06:39:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/2f9e8698a197481992ef6d8684e6ec0e 2024-11-11T06:39:55,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/3e7533b40f9c4641b8890e18caa62d66 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/3e7533b40f9c4641b8890e18caa62d66 2024-11-11T06:39:55,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/3e7533b40f9c4641b8890e18caa62d66, entries=200, sequenceid=93, filesize=14.0 K 2024-11-11T06:39:55,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/9bcc50bb59264fbeaa787bf4dd2fdc03 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9bcc50bb59264fbeaa787bf4dd2fdc03 2024-11-11T06:39:55,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T06:39:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:55,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9bcc50bb59264fbeaa787bf4dd2fdc03, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T06:39:55,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/2f9e8698a197481992ef6d8684e6ec0e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2f9e8698a197481992ef6d8684e6ec0e 2024-11-11T06:39:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T06:39:55,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2f9e8698a197481992ef6d8684e6ec0e, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T06:39:55,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1b6d451b960d07000ebc207f625528a7 in 1120ms, sequenceid=93, compaction requested=false 2024-11-11T06:39:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:55,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T06:39:55,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:55,658 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:39:55,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/c18171fa80d04603a8a788d320220bb5 is 50, key is test_row_0/A:col10/1731307194574/Put/seqid=0 2024-11-11T06:39:55,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741855_1031 (size=12001) 2024-11-11T06:39:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:55,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307255764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307255766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307255769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307255780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307255785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307255889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307255889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307255899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307255899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307256100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307256100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/c18171fa80d04603a8a788d320220bb5 2024-11-11T06:39:56,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307256110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307256110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/39ed3b3534a5429ea0524253cd55ee39 is 50, key is test_row_0/B:col10/1731307194574/Put/seqid=0 2024-11-11T06:39:56,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741856_1032 (size=12001) 2024-11-11T06:39:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307256409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307256409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307256420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307256429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T06:39:56,594 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/39ed3b3534a5429ea0524253cd55ee39 2024-11-11T06:39:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/104ac9bc5a2b4623b73b8d7408b0dddd is 50, key is test_row_0/C:col10/1731307194574/Put/seqid=0 2024-11-11T06:39:56,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741857_1033 (size=12001) 2024-11-11T06:39:56,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307256790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,798 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1b6d451b960d07000ebc207f625528a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:39:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307256930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307256930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307256931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:56,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307256940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:57,042 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/104ac9bc5a2b4623b73b8d7408b0dddd 2024-11-11T06:39:57,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/c18171fa80d04603a8a788d320220bb5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c18171fa80d04603a8a788d320220bb5 2024-11-11T06:39:57,069 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c18171fa80d04603a8a788d320220bb5, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T06:39:57,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/39ed3b3534a5429ea0524253cd55ee39 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/39ed3b3534a5429ea0524253cd55ee39 2024-11-11T06:39:57,084 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/39ed3b3534a5429ea0524253cd55ee39, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T06:39:57,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/104ac9bc5a2b4623b73b8d7408b0dddd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/104ac9bc5a2b4623b73b8d7408b0dddd 2024-11-11T06:39:57,097 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/104ac9bc5a2b4623b73b8d7408b0dddd, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T06:39:57,099 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1b6d451b960d07000ebc207f625528a7 in 1440ms, sequenceid=119, compaction requested=true 2024-11-11T06:39:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T06:39:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T06:39:57,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T06:39:57,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7030 sec 2024-11-11T06:39:57,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.7140 sec 2024-11-11T06:39:57,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:39:57,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:39:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:57,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/8067c5aeec6045cc9e5fc040c942cd6e is 50, key is test_row_0/A:col10/1731307197944/Put/seqid=0 2024-11-11T06:39:58,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741858_1034 (size=14491) 2024-11-11T06:39:58,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307258033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307258036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307258038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307258041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307258149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307258150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307258150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307258150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307258360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307258360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307258360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307258370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/8067c5aeec6045cc9e5fc040c942cd6e 2024-11-11T06:39:58,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2c25c5dda0574b848aaf1f96b34ee737 is 50, key is test_row_0/B:col10/1731307197944/Put/seqid=0 2024-11-11T06:39:58,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741859_1035 (size=12101) 2024-11-11T06:39:58,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2c25c5dda0574b848aaf1f96b34ee737 2024-11-11T06:39:58,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3509c252bb7f482eae7be51d855aba22 is 50, key is test_row_0/C:col10/1731307197944/Put/seqid=0 2024-11-11T06:39:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T06:39:58,550 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:39:58,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:39:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T06:39:58,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741860_1036 (size=12101) 2024-11-11T06:39:58,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:39:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T06:39:58,556 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:39:58,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:39:58,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3509c252bb7f482eae7be51d855aba22 2024-11-11T06:39:58,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/8067c5aeec6045cc9e5fc040c942cd6e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8067c5aeec6045cc9e5fc040c942cd6e 2024-11-11T06:39:58,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8067c5aeec6045cc9e5fc040c942cd6e, entries=200, sequenceid=132, filesize=14.2 K 2024-11-11T06:39:58,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2c25c5dda0574b848aaf1f96b34ee737 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2c25c5dda0574b848aaf1f96b34ee737 2024-11-11T06:39:58,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2c25c5dda0574b848aaf1f96b34ee737, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T06:39:58,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3509c252bb7f482eae7be51d855aba22 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3509c252bb7f482eae7be51d855aba22 2024-11-11T06:39:58,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3509c252bb7f482eae7be51d855aba22, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T06:39:58,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1b6d451b960d07000ebc207f625528a7 in 694ms, sequenceid=132, compaction requested=true 2024-11-11T06:39:58,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:58,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:39:58,644 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:39:58,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:39:58,644 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:39:58,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:39:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:39:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:39:58,646 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:39:58,646 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:39:58,646 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:39:58,646 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:39:58,646 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:58,646 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:58,646 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c5a8a83d19a14a0eb0fd7cdc399608be, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/3e7533b40f9c4641b8890e18caa62d66, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c18171fa80d04603a8a788d320220bb5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8067c5aeec6045cc9e5fc040c942cd6e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=51.7 K 2024-11-11T06:39:58,646 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/44f59547ef3349bdbfc5a4fbccb76bd4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9bcc50bb59264fbeaa787bf4dd2fdc03, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/39ed3b3534a5429ea0524253cd55ee39, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2c25c5dda0574b848aaf1f96b34ee737] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=47.1 K 2024-11-11T06:39:58,647 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5a8a83d19a14a0eb0fd7cdc399608be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731307192154 2024-11-11T06:39:58,647 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f59547ef3349bdbfc5a4fbccb76bd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731307192154 2024-11-11T06:39:58,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:39:58,648 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e7533b40f9c4641b8890e18caa62d66, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731307193329 2024-11-11T06:39:58,648 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bcc50bb59264fbeaa787bf4dd2fdc03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731307193329 2024-11-11T06:39:58,648 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18171fa80d04603a8a788d320220bb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731307194574 2024-11-11T06:39:58,649 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8067c5aeec6045cc9e5fc040c942cd6e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307195762 2024-11-11T06:39:58,649 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ed3b3534a5429ea0524253cd55ee39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731307194574 2024-11-11T06:39:58,649 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c25c5dda0574b848aaf1f96b34ee737, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307195762 2024-11-11T06:39:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T06:39:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:58,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:39:58,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T06:39:58,682 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:39:58,683 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/a9a879a7a1be41259daa8265b3f690d0 is 50, key is test_row_0/A:col10/1731307197944/Put/seqid=0 2024-11-11T06:39:58,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/2a62ce536c8f44d596b461adabcb22c5 is 50, key is test_row_0/A:col10/1731307198033/Put/seqid=0 2024-11-11T06:39:58,690 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:39:58,690 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/272035b8b4f0428c8bb75c1441ddbd1b is 50, key is test_row_0/B:col10/1731307197944/Put/seqid=0 2024-11-11T06:39:58,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741861_1037 (size=12375) 2024-11-11T06:39:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307258699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307258699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307258701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307258701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T06:39:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:58,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:58,721 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/a9a879a7a1be41259daa8265b3f690d0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a9a879a7a1be41259daa8265b3f690d0 2024-11-11T06:39:58,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741862_1038 (size=14541) 2024-11-11T06:39:58,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/2a62ce536c8f44d596b461adabcb22c5 2024-11-11T06:39:58,747 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into a9a879a7a1be41259daa8265b3f690d0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:39:58,747 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:58,747 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=12, startTime=1731307198643; duration=0sec 2024-11-11T06:39:58,747 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:39:58,747 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:39:58,747 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:39:58,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741863_1039 (size=12375) 2024-11-11T06:39:58,763 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:39:58,763 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:39:58,764 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:58,764 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/24f76114a5e2470a883910dc4cdeb250, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2f9e8698a197481992ef6d8684e6ec0e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/104ac9bc5a2b4623b73b8d7408b0dddd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3509c252bb7f482eae7be51d855aba22] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=47.1 K 2024-11-11T06:39:58,765 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f76114a5e2470a883910dc4cdeb250, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731307192154 2024-11-11T06:39:58,766 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f9e8698a197481992ef6d8684e6ec0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731307193329 2024-11-11T06:39:58,769 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 104ac9bc5a2b4623b73b8d7408b0dddd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731307194574 2024-11-11T06:39:58,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/7599c8dbc6bf4a6691fc14807991270c is 50, key is test_row_0/B:col10/1731307198033/Put/seqid=0 2024-11-11T06:39:58,770 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3509c252bb7f482eae7be51d855aba22, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307195762 2024-11-11T06:39:58,779 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/272035b8b4f0428c8bb75c1441ddbd1b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/272035b8b4f0428c8bb75c1441ddbd1b 2024-11-11T06:39:58,790 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into 272035b8b4f0428c8bb75c1441ddbd1b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:39:58,790 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:58,790 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=12, startTime=1731307198644; duration=0sec 2024-11-11T06:39:58,790 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:39:58,790 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:39:58,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741864_1040 (size=12151) 2024-11-11T06:39:58,809 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:39:58,810 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3818a52bf1be4e5fb19b3549819d694c is 50, key is test_row_0/C:col10/1731307197944/Put/seqid=0 2024-11-11T06:39:58,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307258810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307258810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307258811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/7599c8dbc6bf4a6691fc14807991270c 2024-11-11T06:39:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307258819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,827 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1b6d451b960d07000ebc207f625528a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:39:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307258826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:58,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/2564c9972ca144d9aad0d3cd27f3b980 is 50, key is test_row_0/C:col10/1731307198033/Put/seqid=0 2024-11-11T06:39:58,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741865_1041 (size=12375) 2024-11-11T06:39:58,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T06:39:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:58,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:58,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:58,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T06:39:58,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741866_1042 (size=12151) 2024-11-11T06:39:58,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/2564c9972ca144d9aad0d3cd27f3b980 2024-11-11T06:39:58,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/2a62ce536c8f44d596b461adabcb22c5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/2a62ce536c8f44d596b461adabcb22c5 2024-11-11T06:39:58,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/2a62ce536c8f44d596b461adabcb22c5, entries=200, sequenceid=156, filesize=14.2 K 2024-11-11T06:39:58,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/7599c8dbc6bf4a6691fc14807991270c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7599c8dbc6bf4a6691fc14807991270c 2024-11-11T06:39:58,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7599c8dbc6bf4a6691fc14807991270c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T06:39:58,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/2564c9972ca144d9aad0d3cd27f3b980 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2564c9972ca144d9aad0d3cd27f3b980 2024-11-11T06:39:58,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2564c9972ca144d9aad0d3cd27f3b980, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T06:39:58,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 1b6d451b960d07000ebc207f625528a7 in 313ms, sequenceid=156, compaction requested=false 2024-11-11T06:39:58,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:59,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T06:39:59,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:59,021 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:39:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:59,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:59,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dff6b035b8004e73aed2cf6f5e118c9b is 50, key is test_row_0/A:col10/1731307198696/Put/seqid=0 2024-11-11T06:39:59,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741867_1043 (size=12151) 2024-11-11T06:39:59,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307259080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307259087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307259088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307259088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T06:39:59,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307259199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307259199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307259215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307259215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,275 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3818a52bf1be4e5fb19b3549819d694c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3818a52bf1be4e5fb19b3549819d694c 2024-11-11T06:39:59,298 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into 3818a52bf1be4e5fb19b3549819d694c(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:39:59,298 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:59,298 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=12, startTime=1731307198645; duration=0sec 2024-11-11T06:39:59,298 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:39:59,298 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:39:59,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307259409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307259409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307259429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307259429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,470 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dff6b035b8004e73aed2cf6f5e118c9b 2024-11-11T06:39:59,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/92b19d2e35b24131b933cb7fa4cd5751 is 50, key is test_row_0/B:col10/1731307198696/Put/seqid=0 2024-11-11T06:39:59,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741868_1044 (size=12151) 2024-11-11T06:39:59,507 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/92b19d2e35b24131b933cb7fa4cd5751 2024-11-11T06:39:59,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cb550a1b1daf455a9473db207f2fbeec is 50, key is test_row_0/C:col10/1731307198696/Put/seqid=0 2024-11-11T06:39:59,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741869_1045 (size=12151) 2024-11-11T06:39:59,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cb550a1b1daf455a9473db207f2fbeec 2024-11-11T06:39:59,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dff6b035b8004e73aed2cf6f5e118c9b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dff6b035b8004e73aed2cf6f5e118c9b 2024-11-11T06:39:59,576 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dff6b035b8004e73aed2cf6f5e118c9b, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T06:39:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/92b19d2e35b24131b933cb7fa4cd5751 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/92b19d2e35b24131b933cb7fa4cd5751 2024-11-11T06:39:59,612 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/92b19d2e35b24131b933cb7fa4cd5751, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T06:39:59,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cb550a1b1daf455a9473db207f2fbeec as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cb550a1b1daf455a9473db207f2fbeec 2024-11-11T06:39:59,624 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cb550a1b1daf455a9473db207f2fbeec, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T06:39:59,625 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 1b6d451b960d07000ebc207f625528a7 in 604ms, sequenceid=173, compaction requested=true 2024-11-11T06:39:59,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:39:59,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:59,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T06:39:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T06:39:59,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T06:39:59,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-11T06:39:59,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0900 sec 2024-11-11T06:39:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T06:39:59,689 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:39:59,690 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:39:59,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T06:39:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T06:39:59,695 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:39:59,696 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:39:59,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:39:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:39:59,724 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:39:59,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T06:39:59,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/054bdaea7514434aa94591ef70d43f6a is 50, key is test_row_0/A:col10/1731307199722/Put/seqid=0 2024-11-11T06:39:59,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741870_1046 (size=12151) 2024-11-11T06:39:59,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307259775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307259777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307259781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307259781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T06:39:59,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:39:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:39:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:39:59,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:39:59,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307259889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307259889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307259889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:39:59,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:39:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307259900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:39:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:39:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T06:40:00,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307260099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307260099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307260111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307260111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:00,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:00,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/054bdaea7514434aa94591ef70d43f6a 2024-11-11T06:40:00,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c33beecf5aba4ab28292f1554096a87d is 50, key is test_row_0/B:col10/1731307199722/Put/seqid=0 2024-11-11T06:40:00,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741871_1047 (size=12151) 2024-11-11T06:40:00,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c33beecf5aba4ab28292f1554096a87d 2024-11-11T06:40:00,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b0adad4d66a342528fdc99ab98ba2b54 is 50, key is test_row_0/C:col10/1731307199722/Put/seqid=0 2024-11-11T06:40:00,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741872_1048 (size=12151) 2024-11-11T06:40:00,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b0adad4d66a342528fdc99ab98ba2b54 2024-11-11T06:40:00,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/054bdaea7514434aa94591ef70d43f6a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/054bdaea7514434aa94591ef70d43f6a 2024-11-11T06:40:00,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/054bdaea7514434aa94591ef70d43f6a, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T06:40:00,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c33beecf5aba4ab28292f1554096a87d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c33beecf5aba4ab28292f1554096a87d 2024-11-11T06:40:00,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c33beecf5aba4ab28292f1554096a87d, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T06:40:00,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b0adad4d66a342528fdc99ab98ba2b54 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b0adad4d66a342528fdc99ab98ba2b54 2024-11-11T06:40:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T06:40:00,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:00,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b0adad4d66a342528fdc99ab98ba2b54, entries=150, sequenceid=198, filesize=11.9 K 2024-11-11T06:40:00,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:00,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 1b6d451b960d07000ebc207f625528a7 in 605ms, sequenceid=198, compaction requested=true 2024-11-11T06:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:00,329 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:00,329 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:00,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:00,332 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:00,332 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:40:00,332 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,332 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a9a879a7a1be41259daa8265b3f690d0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/2a62ce536c8f44d596b461adabcb22c5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dff6b035b8004e73aed2cf6f5e118c9b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/054bdaea7514434aa94591ef70d43f6a] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=50.0 K 2024-11-11T06:40:00,333 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:00,333 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:40:00,333 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,333 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/272035b8b4f0428c8bb75c1441ddbd1b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7599c8dbc6bf4a6691fc14807991270c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/92b19d2e35b24131b933cb7fa4cd5751, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c33beecf5aba4ab28292f1554096a87d] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=47.7 K 2024-11-11T06:40:00,334 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a879a7a1be41259daa8265b3f690d0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307195762 2024-11-11T06:40:00,335 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 272035b8b4f0428c8bb75c1441ddbd1b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307195762 2024-11-11T06:40:00,335 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a62ce536c8f44d596b461adabcb22c5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731307198029 2024-11-11T06:40:00,336 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting dff6b035b8004e73aed2cf6f5e118c9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731307198696 2024-11-11T06:40:00,336 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7599c8dbc6bf4a6691fc14807991270c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731307198033 2024-11-11T06:40:00,337 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054bdaea7514434aa94591ef70d43f6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731307199086 2024-11-11T06:40:00,337 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b19d2e35b24131b933cb7fa4cd5751, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731307198696 2024-11-11T06:40:00,338 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c33beecf5aba4ab28292f1554096a87d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731307199086 2024-11-11T06:40:00,365 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:00,366 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/24984dc7c16d41ae8d4d0efe8422445d is 50, key is test_row_0/B:col10/1731307199722/Put/seqid=0 2024-11-11T06:40:00,382 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:00,383 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/a427c3c51a5c4d2cb74079ef584d4584 is 50, key is test_row_0/A:col10/1731307199722/Put/seqid=0 2024-11-11T06:40:00,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741874_1050 (size=12561) 2024-11-11T06:40:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:00,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:00,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T06:40:00,414 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/a427c3c51a5c4d2cb74079ef584d4584 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a427c3c51a5c4d2cb74079ef584d4584 2024-11-11T06:40:00,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741873_1049 (size=12561) 2024-11-11T06:40:00,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/bde5d31b5c804f689f0c2045810218df is 50, key is test_row_0/A:col10/1731307199763/Put/seqid=0 2024-11-11T06:40:00,429 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into a427c3c51a5c4d2cb74079ef584d4584(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:00,429 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:00,429 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=12, startTime=1731307200329; duration=0sec 2024-11-11T06:40:00,429 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:00,429 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:40:00,429 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:00,433 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/24984dc7c16d41ae8d4d0efe8422445d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/24984dc7c16d41ae8d4d0efe8422445d 2024-11-11T06:40:00,435 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:00,435 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:40:00,435 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,436 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3818a52bf1be4e5fb19b3549819d694c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2564c9972ca144d9aad0d3cd27f3b980, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cb550a1b1daf455a9473db207f2fbeec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b0adad4d66a342528fdc99ab98ba2b54] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=47.7 K 2024-11-11T06:40:00,437 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3818a52bf1be4e5fb19b3549819d694c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307195762 2024-11-11T06:40:00,439 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2564c9972ca144d9aad0d3cd27f3b980, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731307198033 2024-11-11T06:40:00,440 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb550a1b1daf455a9473db207f2fbeec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731307198696 2024-11-11T06:40:00,444 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0adad4d66a342528fdc99ab98ba2b54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731307199086 2024-11-11T06:40:00,445 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into 24984dc7c16d41ae8d4d0efe8422445d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:00,445 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:00,445 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=12, startTime=1731307200329; duration=0sec 2024-11-11T06:40:00,445 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:00,445 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:40:00,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741875_1051 (size=12151) 2024-11-11T06:40:00,469 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:00,470 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/ce6ce5e6194e4bd6bc81f0509e292721 is 50, key is test_row_0/C:col10/1731307199722/Put/seqid=0 2024-11-11T06:40:00,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307260469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307260473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307260476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307260476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741876_1052 (size=12561) 2024-11-11T06:40:00,515 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/ce6ce5e6194e4bd6bc81f0509e292721 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/ce6ce5e6194e4bd6bc81f0509e292721 2024-11-11T06:40:00,524 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into ce6ce5e6194e4bd6bc81f0509e292721(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:00,524 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:00,524 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=12, startTime=1731307200329; duration=0sec 2024-11-11T06:40:00,524 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:00,524 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:40:00,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307260579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307260579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307260589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307260589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:00,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:00,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:00,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307260789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307260789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307260799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307260799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T06:40:00,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/bde5d31b5c804f689f0c2045810218df 2024-11-11T06:40:00,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/8cd100dd4f034e9b879ba75b9fda92aa is 50, key is test_row_0/B:col10/1731307199763/Put/seqid=0 2024-11-11T06:40:00,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741877_1053 (size=12151) 2024-11-11T06:40:00,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:00,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:00,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:01,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307261099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:01,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:01,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307261110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307261110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307261120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:01,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/8cd100dd4f034e9b879ba75b9fda92aa 2024-11-11T06:40:01,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b14998df7158440f86d4134d182806cd is 50, key is test_row_0/C:col10/1731307199763/Put/seqid=0 2024-11-11T06:40:01,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741878_1054 (size=12151) 2024-11-11T06:40:01,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b14998df7158440f86d4134d182806cd 2024-11-11T06:40:01,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/bde5d31b5c804f689f0c2045810218df as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bde5d31b5c804f689f0c2045810218df 2024-11-11T06:40:01,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bde5d31b5c804f689f0c2045810218df, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T06:40:01,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/8cd100dd4f034e9b879ba75b9fda92aa as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8cd100dd4f034e9b879ba75b9fda92aa 2024-11-11T06:40:01,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8cd100dd4f034e9b879ba75b9fda92aa, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T06:40:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/b14998df7158440f86d4134d182806cd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b14998df7158440f86d4134d182806cd 2024-11-11T06:40:01,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:01,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:01,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:01,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:01,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:01,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b14998df7158440f86d4134d182806cd, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T06:40:01,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 1b6d451b960d07000ebc207f625528a7 in 1008ms, sequenceid=212, compaction requested=false 2024-11-11T06:40:01,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:01,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T06:40:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:01,565 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T06:40:01,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/fec66bb1fd8649c98ed10a8a1b4f36fd is 50, key is test_row_0/A:col10/1731307200472/Put/seqid=0 2024-11-11T06:40:01,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741879_1055 (size=12151) 2024-11-11T06:40:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:01,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:01,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307261634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307261637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307261639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307261639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307261750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307261750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307261750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T06:40:01,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307261959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307261959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307261959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,000 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/fec66bb1fd8649c98ed10a8a1b4f36fd 2024-11-11T06:40:02,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/eb6f28ef94514e6682777cbd5635f15d is 50, key is test_row_0/B:col10/1731307200472/Put/seqid=0 2024-11-11T06:40:02,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741880_1056 (size=12151) 2024-11-11T06:40:02,035 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/eb6f28ef94514e6682777cbd5635f15d 2024-11-11T06:40:02,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/dbdb1b9693ee44218891424e64d64ea1 is 50, key is test_row_0/C:col10/1731307200472/Put/seqid=0 2024-11-11T06:40:02,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741881_1057 (size=12151) 2024-11-11T06:40:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307262269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307262269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307262269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,479 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/dbdb1b9693ee44218891424e64d64ea1 2024-11-11T06:40:02,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/fec66bb1fd8649c98ed10a8a1b4f36fd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/fec66bb1fd8649c98ed10a8a1b4f36fd 2024-11-11T06:40:02,495 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/fec66bb1fd8649c98ed10a8a1b4f36fd, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T06:40:02,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/eb6f28ef94514e6682777cbd5635f15d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/eb6f28ef94514e6682777cbd5635f15d 2024-11-11T06:40:02,504 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/eb6f28ef94514e6682777cbd5635f15d, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T06:40:02,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/dbdb1b9693ee44218891424e64d64ea1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/dbdb1b9693ee44218891424e64d64ea1 2024-11-11T06:40:02,515 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/dbdb1b9693ee44218891424e64d64ea1, entries=150, sequenceid=238, filesize=11.9 K 2024-11-11T06:40:02,517 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 1b6d451b960d07000ebc207f625528a7 in 952ms, sequenceid=238, compaction requested=true 2024-11-11T06:40:02,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:02,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:02,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T06:40:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T06:40:02,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T06:40:02,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8230 sec 2024-11-11T06:40:02,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.8340 sec 2024-11-11T06:40:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:02,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:02,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T06:40:02,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/9e0a5e55a501428f9ead8fa2d8d23876 is 50, key is test_row_0/A:col10/1731307201635/Put/seqid=0 2024-11-11T06:40:02,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741882_1058 (size=14541) 2024-11-11T06:40:02,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/9e0a5e55a501428f9ead8fa2d8d23876 2024-11-11T06:40:02,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1 is 50, key is test_row_0/B:col10/1731307201635/Put/seqid=0 2024-11-11T06:40:02,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741883_1059 (size=12151) 2024-11-11T06:40:02,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1 2024-11-11T06:40:02,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307262784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307262789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307262789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307262789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3bb20259b3334b71833c4507b3cab416 is 50, key is test_row_0/C:col10/1731307201635/Put/seqid=0 2024-11-11T06:40:02,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741884_1060 (size=12151) 2024-11-11T06:40:02,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3bb20259b3334b71833c4507b3cab416 2024-11-11T06:40:02,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/9e0a5e55a501428f9ead8fa2d8d23876 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/9e0a5e55a501428f9ead8fa2d8d23876 2024-11-11T06:40:02,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/9e0a5e55a501428f9ead8fa2d8d23876, entries=200, sequenceid=253, filesize=14.2 K 2024-11-11T06:40:02,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1 2024-11-11T06:40:02,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1, entries=150, sequenceid=253, filesize=11.9 K 2024-11-11T06:40:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3bb20259b3334b71833c4507b3cab416 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3bb20259b3334b71833c4507b3cab416 2024-11-11T06:40:02,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3bb20259b3334b71833c4507b3cab416, entries=150, sequenceid=253, filesize=11.9 K 2024-11-11T06:40:02,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1b6d451b960d07000ebc207f625528a7 in 214ms, sequenceid=253, compaction requested=true 2024-11-11T06:40:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:02,876 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:02,876 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:02,878 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:02,878 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:02,878 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:40:02,878 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:40:02,878 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:02,878 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:02,878 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a427c3c51a5c4d2cb74079ef584d4584, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bde5d31b5c804f689f0c2045810218df, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/fec66bb1fd8649c98ed10a8a1b4f36fd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/9e0a5e55a501428f9ead8fa2d8d23876] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=50.2 K 2024-11-11T06:40:02,878 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/24984dc7c16d41ae8d4d0efe8422445d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8cd100dd4f034e9b879ba75b9fda92aa, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/eb6f28ef94514e6682777cbd5635f15d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=47.9 K 2024-11-11T06:40:02,879 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a427c3c51a5c4d2cb74079ef584d4584, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731307199086 2024-11-11T06:40:02,879 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 24984dc7c16d41ae8d4d0efe8422445d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731307199086 2024-11-11T06:40:02,879 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde5d31b5c804f689f0c2045810218df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731307199763 2024-11-11T06:40:02,879 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cd100dd4f034e9b879ba75b9fda92aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731307199763 2024-11-11T06:40:02,880 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting eb6f28ef94514e6682777cbd5635f15d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731307200460 2024-11-11T06:40:02,880 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting fec66bb1fd8649c98ed10a8a1b4f36fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731307200460 2024-11-11T06:40:02,881 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e0a5e55a501428f9ead8fa2d8d23876, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731307201628 2024-11-11T06:40:02,881 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c3dbb9cdcb21465dbcfdee7ffbe3d8c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731307201634 2024-11-11T06:40:02,885 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:02,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T06:40:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:02,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/7570fab4fc0247f7a0a30e6846d9e243 is 50, key is test_row_0/A:col10/1731307202786/Put/seqid=0 2024-11-11T06:40:02,914 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:02,915 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/783c6f85768e4d80884a7ed73f9199a5 is 50, key is test_row_0/A:col10/1731307201635/Put/seqid=0 2024-11-11T06:40:02,922 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:02,923 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/4ea25e35d66b46f59fe1bb994e23033d is 50, key is test_row_0/B:col10/1731307201635/Put/seqid=0 2024-11-11T06:40:02,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741885_1061 (size=12301) 2024-11-11T06:40:02,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/7570fab4fc0247f7a0a30e6846d9e243 2024-11-11T06:40:02,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307262940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307262942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307262945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741886_1062 (size=12697) 2024-11-11T06:40:03,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2fa4c86fca5644d0b06ec595ccb0a378 is 50, key is test_row_0/B:col10/1731307202786/Put/seqid=0 2024-11-11T06:40:03,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741887_1063 (size=12697) 2024-11-11T06:40:03,040 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/4ea25e35d66b46f59fe1bb994e23033d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4ea25e35d66b46f59fe1bb994e23033d 2024-11-11T06:40:03,051 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into 4ea25e35d66b46f59fe1bb994e23033d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:03,051 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:03,051 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=12, startTime=1731307202876; duration=0sec 2024-11-11T06:40:03,052 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:03,052 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:40:03,052 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:03,054 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:03,054 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:40:03,054 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:03,054 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/ce6ce5e6194e4bd6bc81f0509e292721, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b14998df7158440f86d4134d182806cd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/dbdb1b9693ee44218891424e64d64ea1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3bb20259b3334b71833c4507b3cab416] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=47.9 K 2024-11-11T06:40:03,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307263049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,055 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ce6ce5e6194e4bd6bc81f0509e292721, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731307199086 2024-11-11T06:40:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,056 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b14998df7158440f86d4134d182806cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731307199763 2024-11-11T06:40:03,057 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting dbdb1b9693ee44218891424e64d64ea1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731307200460 2024-11-11T06:40:03,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741888_1064 (size=12301) 2024-11-11T06:40:03,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2fa4c86fca5644d0b06ec595ccb0a378 2024-11-11T06:40:03,059 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb20259b3334b71833c4507b3cab416, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731307201634 2024-11-11T06:40:03,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307263061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307263063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/5a975ce8b4b44d6facf0a990dd4f0cc1 is 50, key is test_row_0/C:col10/1731307202786/Put/seqid=0 2024-11-11T06:40:03,112 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:03,113 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/fcf07021a5eb4de9a793d2ab2c288ea7 is 50, key is test_row_0/C:col10/1731307201635/Put/seqid=0 2024-11-11T06:40:03,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741889_1065 (size=12301) 2024-11-11T06:40:03,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/5a975ce8b4b44d6facf0a990dd4f0cc1 2024-11-11T06:40:03,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/7570fab4fc0247f7a0a30e6846d9e243 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7570fab4fc0247f7a0a30e6846d9e243 2024-11-11T06:40:03,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7570fab4fc0247f7a0a30e6846d9e243, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T06:40:03,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741890_1066 (size=12697) 2024-11-11T06:40:03,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2fa4c86fca5644d0b06ec595ccb0a378 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2fa4c86fca5644d0b06ec595ccb0a378 2024-11-11T06:40:03,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2fa4c86fca5644d0b06ec595ccb0a378, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T06:40:03,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/5a975ce8b4b44d6facf0a990dd4f0cc1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5a975ce8b4b44d6facf0a990dd4f0cc1 2024-11-11T06:40:03,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5a975ce8b4b44d6facf0a990dd4f0cc1, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T06:40:03,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1b6d451b960d07000ebc207f625528a7 in 297ms, sequenceid=275, compaction requested=false 2024-11-11T06:40:03,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:03,264 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:03,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T06:40:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:03,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/cd020605f9b846668013bcae2cb7079e is 50, key is test_row_0/A:col10/1731307202935/Put/seqid=0 2024-11-11T06:40:03,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741891_1067 (size=14741) 2024-11-11T06:40:03,414 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/783c6f85768e4d80884a7ed73f9199a5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/783c6f85768e4d80884a7ed73f9199a5 2024-11-11T06:40:03,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307263408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307263415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307263419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,434 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into 783c6f85768e4d80884a7ed73f9199a5(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:03,434 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:03,434 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=12, startTime=1731307202876; duration=0sec 2024-11-11T06:40:03,434 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:03,434 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:40:03,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307263529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307263529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307263529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,565 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/fcf07021a5eb4de9a793d2ab2c288ea7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fcf07021a5eb4de9a793d2ab2c288ea7 2024-11-11T06:40:03,572 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into fcf07021a5eb4de9a793d2ab2c288ea7(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:03,572 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:03,572 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=12, startTime=1731307202876; duration=0sec 2024-11-11T06:40:03,572 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:03,572 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:40:03,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/cd020605f9b846668013bcae2cb7079e 2024-11-11T06:40:03,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c4eb0ec8ff8e49c39067ed2493d2a5cf is 50, key is test_row_0/B:col10/1731307202935/Put/seqid=0 2024-11-11T06:40:03,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741892_1068 (size=12301) 2024-11-11T06:40:03,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307263739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307263739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307263739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307263809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307263809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:03,810 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1b6d451b960d07000ebc207f625528a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,811 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in 1b6d451b960d07000ebc207f625528a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T06:40:03,860 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:03,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T06:40:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T06:40:03,862 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:03,864 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:03,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T06:40:04,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T06:40:04,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:04,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307264049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307264049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307264049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c4eb0ec8ff8e49c39067ed2493d2a5cf 2024-11-11T06:40:04,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/57b03b22d50445eaa6fc53895622ccba is 50, key is test_row_0/C:col10/1731307202935/Put/seqid=0 2024-11-11T06:40:04,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T06:40:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741893_1069 (size=12301) 2024-11-11T06:40:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/57b03b22d50445eaa6fc53895622ccba 2024-11-11T06:40:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T06:40:04,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/cd020605f9b846668013bcae2cb7079e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/cd020605f9b846668013bcae2cb7079e 2024-11-11T06:40:04,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/cd020605f9b846668013bcae2cb7079e, entries=200, sequenceid=291, filesize=14.4 K 2024-11-11T06:40:04,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c4eb0ec8ff8e49c39067ed2493d2a5cf as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c4eb0ec8ff8e49c39067ed2493d2a5cf 2024-11-11T06:40:04,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c4eb0ec8ff8e49c39067ed2493d2a5cf, entries=150, sequenceid=291, filesize=12.0 K 2024-11-11T06:40:04,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/57b03b22d50445eaa6fc53895622ccba as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/57b03b22d50445eaa6fc53895622ccba 2024-11-11T06:40:04,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/57b03b22d50445eaa6fc53895622ccba, entries=150, sequenceid=291, filesize=12.0 K 2024-11-11T06:40:04,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1b6d451b960d07000ebc207f625528a7 in 963ms, sequenceid=291, compaction requested=true 2024-11-11T06:40:04,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:04,227 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:04,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:04,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:04,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:04,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:04,228 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:04,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:04,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:04,229 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:04,229 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:40:04,229 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,229 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/783c6f85768e4d80884a7ed73f9199a5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7570fab4fc0247f7a0a30e6846d9e243, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/cd020605f9b846668013bcae2cb7079e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=38.8 K 2024-11-11T06:40:04,229 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:04,229 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 783c6f85768e4d80884a7ed73f9199a5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731307201634 2024-11-11T06:40:04,229 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:40:04,229 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,230 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4ea25e35d66b46f59fe1bb994e23033d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2fa4c86fca5644d0b06ec595ccb0a378, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c4eb0ec8ff8e49c39067ed2493d2a5cf] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=36.4 K 2024-11-11T06:40:04,230 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7570fab4fc0247f7a0a30e6846d9e243, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731307202781 2024-11-11T06:40:04,230 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea25e35d66b46f59fe1bb994e23033d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731307201634 2024-11-11T06:40:04,230 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting cd020605f9b846668013bcae2cb7079e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731307202921 2024-11-11T06:40:04,230 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa4c86fca5644d0b06ec595ccb0a378, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731307202781 2024-11-11T06:40:04,231 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4eb0ec8ff8e49c39067ed2493d2a5cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731307202921 2024-11-11T06:40:04,244 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:04,244 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:04,245 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/bf9131e5eba84580ba1a9db843716d2f is 50, key is test_row_0/A:col10/1731307202935/Put/seqid=0 2024-11-11T06:40:04,245 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/d37e3ed888b347839831f8f95c9367fc is 50, key is test_row_0/B:col10/1731307202935/Put/seqid=0 2024-11-11T06:40:04,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741894_1070 (size=12949) 2024-11-11T06:40:04,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741895_1071 (size=12949) 2024-11-11T06:40:04,264 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/bf9131e5eba84580ba1a9db843716d2f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bf9131e5eba84580ba1a9db843716d2f 2024-11-11T06:40:04,273 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/d37e3ed888b347839831f8f95c9367fc as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d37e3ed888b347839831f8f95c9367fc 2024-11-11T06:40:04,273 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into bf9131e5eba84580ba1a9db843716d2f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:04,273 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:04,273 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=13, startTime=1731307204227; duration=0sec 2024-11-11T06:40:04,274 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:04,274 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:40:04,274 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:04,277 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:04,277 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:40:04,277 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,278 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fcf07021a5eb4de9a793d2ab2c288ea7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5a975ce8b4b44d6facf0a990dd4f0cc1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/57b03b22d50445eaa6fc53895622ccba] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=36.4 K 2024-11-11T06:40:04,281 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting fcf07021a5eb4de9a793d2ab2c288ea7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731307201634 2024-11-11T06:40:04,282 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a975ce8b4b44d6facf0a990dd4f0cc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731307202781 2024-11-11T06:40:04,283 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 57b03b22d50445eaa6fc53895622ccba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731307202921 2024-11-11T06:40:04,284 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into d37e3ed888b347839831f8f95c9367fc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:04,284 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:04,284 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=13, startTime=1731307204228; duration=0sec 2024-11-11T06:40:04,284 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:04,284 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:40:04,304 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#59 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:04,304 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/9b5e373339e64265995e8a5271d2753e is 50, key is test_row_0/C:col10/1731307202935/Put/seqid=0 2024-11-11T06:40:04,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741896_1072 (size=12949) 2024-11-11T06:40:04,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T06:40:04,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,325 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/9b5e373339e64265995e8a5271d2753e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/9b5e373339e64265995e8a5271d2753e 2024-11-11T06:40:04,325 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T06:40:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/8ff734cb17f04f4da29f0c1182610788 is 50, key is test_row_0/A:col10/1731307203405/Put/seqid=0 2024-11-11T06:40:04,336 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into 9b5e373339e64265995e8a5271d2753e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:04,336 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:04,336 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=13, startTime=1731307204228; duration=0sec 2024-11-11T06:40:04,336 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:04,336 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:40:04,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741897_1073 (size=12301) 2024-11-11T06:40:04,348 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/8ff734cb17f04f4da29f0c1182610788 2024-11-11T06:40:04,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/a21d4260dc84405db69dc926c29d3e1f is 50, key is test_row_0/B:col10/1731307203405/Put/seqid=0 2024-11-11T06:40:04,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741898_1074 (size=12301) 2024-11-11T06:40:04,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/a21d4260dc84405db69dc926c29d3e1f 2024-11-11T06:40:04,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3e259b2701c04983b42881f1b1479f4e is 50, key is test_row_0/C:col10/1731307203405/Put/seqid=0 2024-11-11T06:40:04,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741899_1075 (size=12301) 2024-11-11T06:40:04,419 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3e259b2701c04983b42881f1b1479f4e 2024-11-11T06:40:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/8ff734cb17f04f4da29f0c1182610788 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8ff734cb17f04f4da29f0c1182610788 2024-11-11T06:40:04,436 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8ff734cb17f04f4da29f0c1182610788, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T06:40:04,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/a21d4260dc84405db69dc926c29d3e1f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/a21d4260dc84405db69dc926c29d3e1f 2024-11-11T06:40:04,445 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/a21d4260dc84405db69dc926c29d3e1f, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T06:40:04,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/3e259b2701c04983b42881f1b1479f4e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3e259b2701c04983b42881f1b1479f4e 2024-11-11T06:40:04,456 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3e259b2701c04983b42881f1b1479f4e, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T06:40:04,457 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 1b6d451b960d07000ebc207f625528a7 in 132ms, sequenceid=316, compaction requested=false 2024-11-11T06:40:04,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:04,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T06:40:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T06:40:04,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T06:40:04,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 597 msec 2024-11-11T06:40:04,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 606 msec 2024-11-11T06:40:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T06:40:04,489 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:04,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T06:40:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T06:40:04,492 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:04,494 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:04,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:04,570 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:04,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:40:04,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/f4bff606eaea4c53a9d7fa7858ce4f8e is 50, key is test_row_0/A:col10/1731307204570/Put/seqid=0 2024-11-11T06:40:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T06:40:04,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741900_1076 (size=19621) 2024-11-11T06:40:04,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307264641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T06:40:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307264643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307264643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307264759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307264759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307264760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T06:40:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T06:40:04,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T06:40:04,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:04,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:04,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:04,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307264969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307264970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307264973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/f4bff606eaea4c53a9d7fa7858ce4f8e 2024-11-11T06:40:05,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/1425c5daa10a41bea90c7ab339510f2b is 50, key is test_row_0/B:col10/1731307204570/Put/seqid=0 2024-11-11T06:40:05,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741901_1077 (size=12301) 2024-11-11T06:40:05,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/1425c5daa10a41bea90c7ab339510f2b 2024-11-11T06:40:05,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cc6307b40cbf4fbd967a4470d253d579 is 50, key is test_row_0/C:col10/1731307204570/Put/seqid=0 2024-11-11T06:40:05,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741902_1078 (size=12301) 2024-11-11T06:40:05,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cc6307b40cbf4fbd967a4470d253d579 2024-11-11T06:40:05,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/f4bff606eaea4c53a9d7fa7858ce4f8e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f4bff606eaea4c53a9d7fa7858ce4f8e 2024-11-11T06:40:05,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f4bff606eaea4c53a9d7fa7858ce4f8e, entries=300, sequenceid=329, filesize=19.2 K 2024-11-11T06:40:05,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/1425c5daa10a41bea90c7ab339510f2b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/1425c5daa10a41bea90c7ab339510f2b 2024-11-11T06:40:05,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T06:40:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:05,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T06:40:05,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/1425c5daa10a41bea90c7ab339510f2b, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T06:40:05,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cc6307b40cbf4fbd967a4470d253d579 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cc6307b40cbf4fbd967a4470d253d579 2024-11-11T06:40:05,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cc6307b40cbf4fbd967a4470d253d579, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T06:40:05,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1b6d451b960d07000ebc207f625528a7 in 573ms, sequenceid=329, compaction requested=true 2024-11-11T06:40:05,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:05,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:05,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:05,145 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:05,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:05,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:05,145 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:05,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:05,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:05,147 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:05,147 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:05,149 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:40:05,149 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:40:05,149 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:05,150 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:05,150 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d37e3ed888b347839831f8f95c9367fc, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/a21d4260dc84405db69dc926c29d3e1f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/1425c5daa10a41bea90c7ab339510f2b] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=36.7 K 2024-11-11T06:40:05,150 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bf9131e5eba84580ba1a9db843716d2f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8ff734cb17f04f4da29f0c1182610788, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f4bff606eaea4c53a9d7fa7858ce4f8e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=43.8 K 2024-11-11T06:40:05,151 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting d37e3ed888b347839831f8f95c9367fc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731307202921 2024-11-11T06:40:05,152 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9131e5eba84580ba1a9db843716d2f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731307202921 2024-11-11T06:40:05,153 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting a21d4260dc84405db69dc926c29d3e1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731307203401 2024-11-11T06:40:05,153 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff734cb17f04f4da29f0c1182610788, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731307203401 2024-11-11T06:40:05,154 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 1425c5daa10a41bea90c7ab339510f2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307204564 2024-11-11T06:40:05,154 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4bff606eaea4c53a9d7fa7858ce4f8e, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307204561 2024-11-11T06:40:05,170 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:05,170 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/af49bf8f9f6e4d0a9da0ff9c06599000 is 50, key is test_row_0/A:col10/1731307204570/Put/seqid=0 2024-11-11T06:40:05,180 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:05,181 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/d84b766389e1403387633189c3be9c52 is 50, key is test_row_0/B:col10/1731307204570/Put/seqid=0 2024-11-11T06:40:05,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741903_1079 (size=13051) 2024-11-11T06:40:05,219 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/af49bf8f9f6e4d0a9da0ff9c06599000 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/af49bf8f9f6e4d0a9da0ff9c06599000 2024-11-11T06:40:05,228 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into af49bf8f9f6e4d0a9da0ff9c06599000(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:05,228 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:05,228 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=13, startTime=1731307205145; duration=0sec 2024-11-11T06:40:05,228 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:05,228 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:40:05,228 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:05,231 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:05,231 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:40:05,231 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:05,232 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/9b5e373339e64265995e8a5271d2753e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3e259b2701c04983b42881f1b1479f4e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cc6307b40cbf4fbd967a4470d253d579] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=36.7 K 2024-11-11T06:40:05,232 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5e373339e64265995e8a5271d2753e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731307202921 2024-11-11T06:40:05,233 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e259b2701c04983b42881f1b1479f4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731307203401 2024-11-11T06:40:05,234 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc6307b40cbf4fbd967a4470d253d579, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307204564 2024-11-11T06:40:05,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741904_1080 (size=13051) 2024-11-11T06:40:05,249 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#68 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:05,251 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/795f219922304a989362eff03be6142d is 50, key is test_row_0/C:col10/1731307204570/Put/seqid=0 2024-11-11T06:40:05,256 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/d84b766389e1403387633189c3be9c52 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d84b766389e1403387633189c3be9c52 2024-11-11T06:40:05,265 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into d84b766389e1403387633189c3be9c52(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:05,265 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:05,266 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=13, startTime=1731307205145; duration=0sec 2024-11-11T06:40:05,266 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:05,266 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:40:05,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T06:40:05,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:05,267 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T06:40:05,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/721e10a79b764609b05a7ebfdace78ba is 50, key is test_row_0/A:col10/1731307204595/Put/seqid=0 2024-11-11T06:40:05,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741905_1081 (size=13051) 2024-11-11T06:40:05,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307265310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307265311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741906_1082 (size=12301) 2024-11-11T06:40:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307265312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307265429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307265430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307265430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T06:40:05,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307265639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307265639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307265639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,682 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/795f219922304a989362eff03be6142d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/795f219922304a989362eff03be6142d 2024-11-11T06:40:05,695 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into 795f219922304a989362eff03be6142d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:05,695 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:05,695 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=13, startTime=1731307205145; duration=0sec 2024-11-11T06:40:05,698 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:05,698 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:40:05,721 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/721e10a79b764609b05a7ebfdace78ba 2024-11-11T06:40:05,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/4fbaddf035cf43df94d127b5d9296177 is 50, key is test_row_0/B:col10/1731307204595/Put/seqid=0 2024-11-11T06:40:05,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741907_1083 (size=12301) 2024-11-11T06:40:05,775 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/4fbaddf035cf43df94d127b5d9296177 2024-11-11T06:40:05,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/fb7a60c5d297492b9211c31cdbccef43 is 50, key is test_row_0/C:col10/1731307204595/Put/seqid=0 2024-11-11T06:40:05,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741908_1084 (size=12301) 2024-11-11T06:40:05,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307265829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307265829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,832 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in 1b6d451b960d07000ebc207f625528a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,833 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1b6d451b960d07000ebc207f625528a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307265949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307265949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:05,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307265949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,221 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/fb7a60c5d297492b9211c31cdbccef43 2024-11-11T06:40:06,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/721e10a79b764609b05a7ebfdace78ba as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/721e10a79b764609b05a7ebfdace78ba 2024-11-11T06:40:06,235 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/721e10a79b764609b05a7ebfdace78ba, entries=150, sequenceid=355, filesize=12.0 K 2024-11-11T06:40:06,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/4fbaddf035cf43df94d127b5d9296177 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4fbaddf035cf43df94d127b5d9296177 2024-11-11T06:40:06,244 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4fbaddf035cf43df94d127b5d9296177, entries=150, sequenceid=355, filesize=12.0 K 2024-11-11T06:40:06,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/fb7a60c5d297492b9211c31cdbccef43 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fb7a60c5d297492b9211c31cdbccef43 2024-11-11T06:40:06,260 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fb7a60c5d297492b9211c31cdbccef43, entries=150, sequenceid=355, filesize=12.0 K 2024-11-11T06:40:06,262 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1b6d451b960d07000ebc207f625528a7 in 995ms, sequenceid=355, compaction requested=false 2024-11-11T06:40:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:06,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T06:40:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T06:40:06,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T06:40:06,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7700 sec 2024-11-11T06:40:06,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7770 sec 2024-11-11T06:40:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:06,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:06,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:40:06,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/e152aa77f8e84ed0976885c0f2553fa4 is 50, key is test_row_0/A:col10/1731307206459/Put/seqid=0 2024-11-11T06:40:06,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741909_1085 (size=9857) 2024-11-11T06:40:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307266494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307266494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307266495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307266599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307266599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307266609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T06:40:06,650 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:06,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T06:40:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T06:40:06,652 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:06,653 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:06,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T06:40:06,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T06:40:06,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:06,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:06,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:06,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307266809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307266809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307266819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:06,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/e152aa77f8e84ed0976885c0f2553fa4 2024-11-11T06:40:06,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/f414109995044e3e8144d5b5a35a4c58 is 50, key is test_row_0/B:col10/1731307206459/Put/seqid=0 2024-11-11T06:40:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741910_1086 (size=9857) 2024-11-11T06:40:06,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/f414109995044e3e8144d5b5a35a4c58 2024-11-11T06:40:06,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/6dd8b6159c8a4a4897c479b564795d01 is 50, key is test_row_0/C:col10/1731307206459/Put/seqid=0 2024-11-11T06:40:06,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741911_1087 (size=9857) 2024-11-11T06:40:06,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T06:40:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:06,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T06:40:07,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T06:40:07,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:07,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:07,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:07,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:07,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307267119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307267119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307267129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T06:40:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:07,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T06:40:07,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/6dd8b6159c8a4a4897c479b564795d01 2024-11-11T06:40:07,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/e152aa77f8e84ed0976885c0f2553fa4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/e152aa77f8e84ed0976885c0f2553fa4 2024-11-11T06:40:07,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/e152aa77f8e84ed0976885c0f2553fa4, entries=100, sequenceid=370, filesize=9.6 K 2024-11-11T06:40:07,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/f414109995044e3e8144d5b5a35a4c58 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f414109995044e3e8144d5b5a35a4c58 2024-11-11T06:40:07,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f414109995044e3e8144d5b5a35a4c58, entries=100, sequenceid=370, filesize=9.6 K 2024-11-11T06:40:07,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/6dd8b6159c8a4a4897c479b564795d01 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/6dd8b6159c8a4a4897c479b564795d01 2024-11-11T06:40:07,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/6dd8b6159c8a4a4897c479b564795d01, entries=100, sequenceid=370, filesize=9.6 K 2024-11-11T06:40:07,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1b6d451b960d07000ebc207f625528a7 in 932ms, sequenceid=370, compaction requested=true 2024-11-11T06:40:07,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:07,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:07,394 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:07,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:07,394 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:07,396 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:07,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:07,396 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:40:07,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:07,396 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:07,396 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d84b766389e1403387633189c3be9c52, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4fbaddf035cf43df94d127b5d9296177, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f414109995044e3e8144d5b5a35a4c58] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=34.4 K 2024-11-11T06:40:07,397 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:07,397 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:40:07,397 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:07,397 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/af49bf8f9f6e4d0a9da0ff9c06599000, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/721e10a79b764609b05a7ebfdace78ba, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/e152aa77f8e84ed0976885c0f2553fa4] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=34.4 K 2024-11-11T06:40:07,397 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting d84b766389e1403387633189c3be9c52, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307204564 2024-11-11T06:40:07,398 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting af49bf8f9f6e4d0a9da0ff9c06599000, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307204564 2024-11-11T06:40:07,398 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fbaddf035cf43df94d127b5d9296177, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731307204595 2024-11-11T06:40:07,398 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721e10a79b764609b05a7ebfdace78ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731307204595 2024-11-11T06:40:07,398 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting e152aa77f8e84ed0976885c0f2553fa4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731307206459 2024-11-11T06:40:07,398 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting f414109995044e3e8144d5b5a35a4c58, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731307206459 2024-11-11T06:40:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:07,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T06:40:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:07,426 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T06:40:07,428 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:07,429 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/00cad3ed35a5453fa12ced83f70e4419 is 50, key is test_row_0/A:col10/1731307206459/Put/seqid=0 2024-11-11T06:40:07,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dfa25a6d2f53426392ee45f7b3251a1a is 50, key is test_row_0/A:col10/1731307206493/Put/seqid=0 2024-11-11T06:40:07,452 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:07,453 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/f0cf597fd5cd4d879ec3403e21151dba is 50, key is test_row_0/B:col10/1731307206459/Put/seqid=0 2024-11-11T06:40:07,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741912_1088 (size=13153) 2024-11-11T06:40:07,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741913_1089 (size=12301) 2024-11-11T06:40:07,501 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dfa25a6d2f53426392ee45f7b3251a1a 2024-11-11T06:40:07,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741914_1090 (size=13153) 2024-11-11T06:40:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/246a25f6428547a0a1b94518fbeba32e is 50, key is test_row_0/B:col10/1731307206493/Put/seqid=0 2024-11-11T06:40:07,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741915_1091 (size=12301) 2024-11-11T06:40:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:07,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:07,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307267644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307267646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307267645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307267759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307267759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307267759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T06:40:07,896 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/00cad3ed35a5453fa12ced83f70e4419 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/00cad3ed35a5453fa12ced83f70e4419 2024-11-11T06:40:07,902 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into 00cad3ed35a5453fa12ced83f70e4419(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:07,902 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:07,902 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=13, startTime=1731307207394; duration=0sec 2024-11-11T06:40:07,902 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:07,902 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:40:07,902 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:07,905 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:07,905 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:40:07,905 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:07,905 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/795f219922304a989362eff03be6142d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fb7a60c5d297492b9211c31cdbccef43, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/6dd8b6159c8a4a4897c479b564795d01] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=34.4 K 2024-11-11T06:40:07,906 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 795f219922304a989362eff03be6142d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307204564 2024-11-11T06:40:07,911 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb7a60c5d297492b9211c31cdbccef43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731307204595 2024-11-11T06:40:07,911 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dd8b6159c8a4a4897c479b564795d01, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731307206459 2024-11-11T06:40:07,918 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/f0cf597fd5cd4d879ec3403e21151dba as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f0cf597fd5cd4d879ec3403e21151dba 2024-11-11T06:40:07,936 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into f0cf597fd5cd4d879ec3403e21151dba(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:07,937 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:07,937 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=13, startTime=1731307207394; duration=0sec 2024-11-11T06:40:07,937 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:07,937 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:40:07,946 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/246a25f6428547a0a1b94518fbeba32e 2024-11-11T06:40:07,956 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:07,957 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/aa24a14473a141349df3cc3997559837 is 50, key is test_row_0/C:col10/1731307206459/Put/seqid=0 2024-11-11T06:40:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307267969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307267969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307267970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cbd3edff73c348baa766eee337665ade is 50, key is test_row_0/C:col10/1731307206493/Put/seqid=0 2024-11-11T06:40:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:07,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741916_1092 (size=13153) 2024-11-11T06:40:07,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741917_1093 (size=12301) 2024-11-11T06:40:07,998 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cbd3edff73c348baa766eee337665ade 2024-11-11T06:40:08,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/dfa25a6d2f53426392ee45f7b3251a1a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dfa25a6d2f53426392ee45f7b3251a1a 2024-11-11T06:40:08,009 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dfa25a6d2f53426392ee45f7b3251a1a, entries=150, sequenceid=392, filesize=12.0 K 2024-11-11T06:40:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/246a25f6428547a0a1b94518fbeba32e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/246a25f6428547a0a1b94518fbeba32e 2024-11-11T06:40:08,018 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/246a25f6428547a0a1b94518fbeba32e, entries=150, sequenceid=392, filesize=12.0 K 2024-11-11T06:40:08,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/cbd3edff73c348baa766eee337665ade as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cbd3edff73c348baa766eee337665ade 2024-11-11T06:40:08,024 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cbd3edff73c348baa766eee337665ade, entries=150, sequenceid=392, filesize=12.0 K 2024-11-11T06:40:08,025 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1b6d451b960d07000ebc207f625528a7 in 599ms, sequenceid=392, compaction requested=false 2024-11-11T06:40:08,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:08,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:08,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T06:40:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T06:40:08,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T06:40:08,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3740 sec 2024-11-11T06:40:08,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3790 sec 2024-11-11T06:40:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:08,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:08,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T06:40:08,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/ea0d52ce28ad4ccca66804d5278faf72 is 50, key is test_row_0/A:col10/1731307207643/Put/seqid=0 2024-11-11T06:40:08,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741918_1094 (size=17181) 2024-11-11T06:40:08,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/ea0d52ce28ad4ccca66804d5278faf72 2024-11-11T06:40:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307268316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307268318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307268326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c8f43ea3efdd4685b094fddbabe0e7a7 is 50, key is test_row_0/B:col10/1731307207643/Put/seqid=0 2024-11-11T06:40:08,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741919_1095 (size=12301) 2024-11-11T06:40:08,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c8f43ea3efdd4685b094fddbabe0e7a7 2024-11-11T06:40:08,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/0147cccac6cd4ebaa3dad444a97c0690 is 50, key is test_row_0/C:col10/1731307207643/Put/seqid=0 2024-11-11T06:40:08,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741920_1096 (size=12301) 2024-11-11T06:40:08,400 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/aa24a14473a141349df3cc3997559837 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/aa24a14473a141349df3cc3997559837 2024-11-11T06:40:08,407 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into aa24a14473a141349df3cc3997559837(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:08,407 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:08,407 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=13, startTime=1731307207399; duration=0sec 2024-11-11T06:40:08,407 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:08,407 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:40:08,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307268429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307268439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307268439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307268638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307268649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307268649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/0147cccac6cd4ebaa3dad444a97c0690 2024-11-11T06:40:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/ea0d52ce28ad4ccca66804d5278faf72 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/ea0d52ce28ad4ccca66804d5278faf72 2024-11-11T06:40:08,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/ea0d52ce28ad4ccca66804d5278faf72, entries=250, sequenceid=410, filesize=16.8 K 2024-11-11T06:40:08,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/c8f43ea3efdd4685b094fddbabe0e7a7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c8f43ea3efdd4685b094fddbabe0e7a7 2024-11-11T06:40:08,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c8f43ea3efdd4685b094fddbabe0e7a7, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T06:40:08,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/0147cccac6cd4ebaa3dad444a97c0690 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/0147cccac6cd4ebaa3dad444a97c0690 2024-11-11T06:40:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T06:40:08,800 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:08,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T06:40:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T06:40:08,802 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:08,803 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:08,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:08,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/0147cccac6cd4ebaa3dad444a97c0690, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T06:40:08,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1b6d451b960d07000ebc207f625528a7 in 528ms, sequenceid=410, compaction requested=true 2024-11-11T06:40:08,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:08,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:08,809 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:08,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:08,810 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:08,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:08,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:08,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:08,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:08,811 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:08,811 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:40:08,811 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:08,811 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/00cad3ed35a5453fa12ced83f70e4419, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dfa25a6d2f53426392ee45f7b3251a1a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/ea0d52ce28ad4ccca66804d5278faf72] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=41.6 K 2024-11-11T06:40:08,812 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:08,812 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:40:08,812 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:08,812 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f0cf597fd5cd4d879ec3403e21151dba, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/246a25f6428547a0a1b94518fbeba32e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c8f43ea3efdd4685b094fddbabe0e7a7] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=36.9 K 2024-11-11T06:40:08,812 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00cad3ed35a5453fa12ced83f70e4419, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731307204636 2024-11-11T06:40:08,813 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfa25a6d2f53426392ee45f7b3251a1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731307206483 2024-11-11T06:40:08,813 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting f0cf597fd5cd4d879ec3403e21151dba, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731307204636 2024-11-11T06:40:08,813 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea0d52ce28ad4ccca66804d5278faf72, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731307207643 2024-11-11T06:40:08,813 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 246a25f6428547a0a1b94518fbeba32e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731307206483 2024-11-11T06:40:08,814 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f43ea3efdd4685b094fddbabe0e7a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731307207643 2024-11-11T06:40:08,827 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:08,827 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:08,828 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/a294449713384808ba511c2e6dcf81f5 is 50, key is test_row_0/A:col10/1731307207643/Put/seqid=0 2024-11-11T06:40:08,829 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/5bf0551b559b4ed4ac59ba0c03f227fd is 50, key is test_row_0/B:col10/1731307207643/Put/seqid=0 2024-11-11T06:40:08,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741921_1097 (size=13255) 2024-11-11T06:40:08,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741922_1098 (size=13255) 2024-11-11T06:40:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T06:40:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:08,954 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:08,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T06:40:08,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T06:40:08,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:08,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:08,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:08,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:08,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:08,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/7de5137649de437994f85a881b0cdaa0 is 50, key is test_row_0/A:col10/1731307208951/Put/seqid=0 2024-11-11T06:40:08,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741923_1099 (size=14741) 2024-11-11T06:40:08,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/7de5137649de437994f85a881b0cdaa0 2024-11-11T06:40:08,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307268974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307268976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307268977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:08,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/920de46362dc4bfd8dd4c70bc24fa529 is 50, key is test_row_0/B:col10/1731307208951/Put/seqid=0 2024-11-11T06:40:09,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741924_1100 (size=12301) 2024-11-11T06:40:09,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307269089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307269089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307269090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T06:40:09,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:09,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T06:40:09,245 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/5bf0551b559b4ed4ac59ba0c03f227fd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/5bf0551b559b4ed4ac59ba0c03f227fd 2024-11-11T06:40:09,263 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into 5bf0551b559b4ed4ac59ba0c03f227fd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:09,263 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:09,263 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=13, startTime=1731307208810; duration=0sec 2024-11-11T06:40:09,263 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:09,263 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:40:09,263 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:09,264 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/a294449713384808ba511c2e6dcf81f5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a294449713384808ba511c2e6dcf81f5 2024-11-11T06:40:09,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T06:40:09,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:09,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,268 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:09,268 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:40:09,269 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,269 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/aa24a14473a141349df3cc3997559837, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cbd3edff73c348baa766eee337665ade, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/0147cccac6cd4ebaa3dad444a97c0690] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=36.9 K 2024-11-11T06:40:09,270 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting aa24a14473a141349df3cc3997559837, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731307204636 2024-11-11T06:40:09,271 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd3edff73c348baa766eee337665ade, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731307206483 2024-11-11T06:40:09,272 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0147cccac6cd4ebaa3dad444a97c0690, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731307207643 2024-11-11T06:40:09,283 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into a294449713384808ba511c2e6dcf81f5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:09,283 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:09,283 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=13, startTime=1731307208809; duration=0sec 2024-11-11T06:40:09,283 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:09,283 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:40:09,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307269299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307269299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307269299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,308 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:09,308 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/56b7b17ccfc044858cb9600f973bc9ad is 50, key is test_row_0/C:col10/1731307207643/Put/seqid=0 2024-11-11T06:40:09,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741925_1101 (size=13255) 2024-11-11T06:40:09,361 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/56b7b17ccfc044858cb9600f973bc9ad as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/56b7b17ccfc044858cb9600f973bc9ad 2024-11-11T06:40:09,372 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into 56b7b17ccfc044858cb9600f973bc9ad(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:09,373 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:09,373 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=13, startTime=1731307208810; duration=0sec 2024-11-11T06:40:09,373 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:09,373 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:40:09,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/920de46362dc4bfd8dd4c70bc24fa529 2024-11-11T06:40:09,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/be040aff8b16443ab299b5b4cd73c892 is 50, key is test_row_0/C:col10/1731307208951/Put/seqid=0 2024-11-11T06:40:09,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T06:40:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T06:40:09,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741926_1102 (size=12301) 2024-11-11T06:40:09,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T06:40:09,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:09,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307269608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307269608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307269608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T06:40:09,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:09,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/be040aff8b16443ab299b5b4cd73c892 2024-11-11T06:40:09,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/7de5137649de437994f85a881b0cdaa0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7de5137649de437994f85a881b0cdaa0 2024-11-11T06:40:09,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7de5137649de437994f85a881b0cdaa0, entries=200, sequenceid=432, filesize=14.4 K 2024-11-11T06:40:09,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/920de46362dc4bfd8dd4c70bc24fa529 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/920de46362dc4bfd8dd4c70bc24fa529 2024-11-11T06:40:09,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/920de46362dc4bfd8dd4c70bc24fa529, entries=150, sequenceid=432, filesize=12.0 K 2024-11-11T06:40:09,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/be040aff8b16443ab299b5b4cd73c892 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/be040aff8b16443ab299b5b4cd73c892 2024-11-11T06:40:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42194 deadline: 1731307269859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,861 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1b6d451b960d07000ebc207f625528a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:09,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/be040aff8b16443ab299b5b4cd73c892, entries=150, sequenceid=432, filesize=12.0 K 2024-11-11T06:40:09,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 1b6d451b960d07000ebc207f625528a7 in 913ms, sequenceid=432, compaction requested=false 2024-11-11T06:40:09,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:09,871 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:09,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T06:40:09,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/75a7030a6911492b86b167ff332df35e is 50, key is test_row_0/A:col10/1731307208975/Put/seqid=0 2024-11-11T06:40:09,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T06:40:09,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:09,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:09,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:09,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741927_1103 (size=14741) 2024-11-11T06:40:09,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/75a7030a6911492b86b167ff332df35e 2024-11-11T06:40:09,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/9529b9a50f094980be4de0e4e4241364 is 50, key is test_row_0/B:col10/1731307208975/Put/seqid=0 2024-11-11T06:40:09,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741928_1104 (size=12301) 2024-11-11T06:40:09,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/9529b9a50f094980be4de0e4e4241364 2024-11-11T06:40:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T06:40:09,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/1429c01d5b2543d3a6e3cb127ef0adc6 is 50, key is test_row_0/C:col10/1731307208975/Put/seqid=0 2024-11-11T06:40:09,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307269948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:09,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:09,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741929_1105 (size=12301) 2024-11-11T06:40:09,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/1429c01d5b2543d3a6e3cb127ef0adc6 2024-11-11T06:40:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/75a7030a6911492b86b167ff332df35e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/75a7030a6911492b86b167ff332df35e 2024-11-11T06:40:09,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/75a7030a6911492b86b167ff332df35e, entries=200, sequenceid=451, filesize=14.4 K 2024-11-11T06:40:09,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/9529b9a50f094980be4de0e4e4241364 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9529b9a50f094980be4de0e4e4241364 2024-11-11T06:40:09,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9529b9a50f094980be4de0e4e4241364, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T06:40:09,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/1429c01d5b2543d3a6e3cb127ef0adc6 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/1429c01d5b2543d3a6e3cb127ef0adc6 2024-11-11T06:40:10,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/1429c01d5b2543d3a6e3cb127ef0adc6, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T06:40:10,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1b6d451b960d07000ebc207f625528a7 in 131ms, sequenceid=451, compaction requested=true 2024-11-11T06:40:10,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:10,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:10,003 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:10,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:10,003 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:10,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:10,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:10,005 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:10,005 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:10,005 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:40:10,005 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:40:10,005 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:10,005 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:10,005 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/5bf0551b559b4ed4ac59ba0c03f227fd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/920de46362dc4bfd8dd4c70bc24fa529, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9529b9a50f094980be4de0e4e4241364] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=37.0 K 2024-11-11T06:40:10,005 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a294449713384808ba511c2e6dcf81f5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7de5137649de437994f85a881b0cdaa0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/75a7030a6911492b86b167ff332df35e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=41.7 K 2024-11-11T06:40:10,006 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a294449713384808ba511c2e6dcf81f5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731307207643 2024-11-11T06:40:10,007 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bf0551b559b4ed4ac59ba0c03f227fd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731307207643 2024-11-11T06:40:10,008 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 920de46362dc4bfd8dd4c70bc24fa529, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731307208313 2024-11-11T06:40:10,008 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de5137649de437994f85a881b0cdaa0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731307208313 2024-11-11T06:40:10,008 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 9529b9a50f094980be4de0e4e4241364, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731307208974 2024-11-11T06:40:10,008 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75a7030a6911492b86b167ff332df35e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731307208973 2024-11-11T06:40:10,016 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:10,016 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/50d26d29951a40a897433f1a53a6871e is 50, key is test_row_0/B:col10/1731307208975/Put/seqid=0 2024-11-11T06:40:10,023 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:10,024 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/81667508dd9f478990c88319964f2b2f is 50, key is test_row_0/A:col10/1731307208975/Put/seqid=0 2024-11-11T06:40:10,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741931_1107 (size=13357) 2024-11-11T06:40:10,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741930_1106 (size=13357) 2024-11-11T06:40:10,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T06:40:10,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:10,037 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T06:40:10,042 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/81667508dd9f478990c88319964f2b2f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/81667508dd9f478990c88319964f2b2f 2024-11-11T06:40:10,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/21e842f285f74120b77c69f84c34ea0d is 50, key is test_row_0/A:col10/1731307209934/Put/seqid=0 2024-11-11T06:40:10,047 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/50d26d29951a40a897433f1a53a6871e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/50d26d29951a40a897433f1a53a6871e 2024-11-11T06:40:10,051 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into 81667508dd9f478990c88319964f2b2f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:10,051 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:10,051 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=13, startTime=1731307210003; duration=0sec 2024-11-11T06:40:10,051 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:10,051 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:40:10,051 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:10,053 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:10,053 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:40:10,053 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:10,053 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/56b7b17ccfc044858cb9600f973bc9ad, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/be040aff8b16443ab299b5b4cd73c892, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/1429c01d5b2543d3a6e3cb127ef0adc6] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=37.0 K 2024-11-11T06:40:10,053 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56b7b17ccfc044858cb9600f973bc9ad, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731307207643 2024-11-11T06:40:10,054 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting be040aff8b16443ab299b5b4cd73c892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731307208313 2024-11-11T06:40:10,054 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1429c01d5b2543d3a6e3cb127ef0adc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731307208974 2024-11-11T06:40:10,055 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into 50d26d29951a40a897433f1a53a6871e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:10,055 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:10,055 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=13, startTime=1731307210003; duration=0sec 2024-11-11T06:40:10,056 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:10,056 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:40:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:10,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. as already flushing 2024-11-11T06:40:10,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741932_1108 (size=12301) 2024-11-11T06:40:10,079 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:10,079 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/489d9f7796ea48a49785854c8d3f33ea is 50, key is test_row_0/C:col10/1731307208975/Put/seqid=0 2024-11-11T06:40:10,084 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/21e842f285f74120b77c69f84c34ea0d 2024-11-11T06:40:10,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741933_1109 (size=13357) 2024-11-11T06:40:10,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/8be9d578e0424e4ba4c600f08c734ba0 is 50, key is test_row_0/B:col10/1731307209934/Put/seqid=0 2024-11-11T06:40:10,109 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/489d9f7796ea48a49785854c8d3f33ea as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/489d9f7796ea48a49785854c8d3f33ea 2024-11-11T06:40:10,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741934_1110 (size=12301) 2024-11-11T06:40:10,123 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/8be9d578e0424e4ba4c600f08c734ba0 2024-11-11T06:40:10,123 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into 489d9f7796ea48a49785854c8d3f33ea(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:10,123 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:10,123 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=13, startTime=1731307210003; duration=0sec 2024-11-11T06:40:10,123 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:10,123 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:40:10,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42242 deadline: 1731307270119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/4c1fa36b6a7d4730a66e51e6627c4811 is 50, key is test_row_0/C:col10/1731307209934/Put/seqid=0 2024-11-11T06:40:10,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42270 deadline: 1731307270124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:10,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42210 deadline: 1731307270127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:10,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:10,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42262 deadline: 1731307270125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:10,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741935_1111 (size=12301) 2024-11-11T06:40:10,164 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/4c1fa36b6a7d4730a66e51e6627c4811 2024-11-11T06:40:10,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/21e842f285f74120b77c69f84c34ea0d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/21e842f285f74120b77c69f84c34ea0d 2024-11-11T06:40:10,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/21e842f285f74120b77c69f84c34ea0d, entries=150, sequenceid=471, filesize=12.0 K 2024-11-11T06:40:10,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/8be9d578e0424e4ba4c600f08c734ba0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8be9d578e0424e4ba4c600f08c734ba0 2024-11-11T06:40:10,201 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8be9d578e0424e4ba4c600f08c734ba0, entries=150, sequenceid=471, filesize=12.0 K 2024-11-11T06:40:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/4c1fa36b6a7d4730a66e51e6627c4811 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/4c1fa36b6a7d4730a66e51e6627c4811 2024-11-11T06:40:10,208 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/4c1fa36b6a7d4730a66e51e6627c4811, entries=150, sequenceid=471, filesize=12.0 K 2024-11-11T06:40:10,209 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T06:40:10,209 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T06:40:10,210 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1b6d451b960d07000ebc207f625528a7 in 172ms, sequenceid=471, compaction requested=false 2024-11-11T06:40:10,210 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:10,210 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:10,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:10,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:10,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T06:40:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T06:40:10,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T06:40:10,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-11T06:40:10,214 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T06:40:10,214 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:10,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4140 sec 2024-11-11T06:40:10,216 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,216 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,216 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,216 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,216 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,216 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:10,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:10,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:10,218 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T06:40:10,218 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:10,218 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,218 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:10,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:10,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-11T06:40:10,241 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T06:40:10,241 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T06:40:10,241 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:10,241 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:10,241 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,241 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,241 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,241 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:10,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:10,241 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T06:40:10,241 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:10,242 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,242 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:10,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:10,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/176fbd0e93a44cf8be18482c2599b26c is 50, key is test_row_0/A:col10/1731307210239/Put/seqid=0 2024-11-11T06:40:10,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741936_1112 (size=12301) 2024-11-11T06:40:10,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/176fbd0e93a44cf8be18482c2599b26c 2024-11-11T06:40:10,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e is 50, key is test_row_0/B:col10/1731307210239/Put/seqid=0 2024-11-11T06:40:10,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741937_1113 (size=12301) 2024-11-11T06:40:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T06:40:10,951 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:11,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e 2024-11-11T06:40:11,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/5665d04f11f042acae2fbaf02eb89814 is 50, key is test_row_0/C:col10/1731307210239/Put/seqid=0 2024-11-11T06:40:11,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741938_1114 (size=12301) 2024-11-11T06:40:11,141 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T06:40:11,141 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:11,141 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:11,141 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:11,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:11,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/5665d04f11f042acae2fbaf02eb89814 2024-11-11T06:40:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/176fbd0e93a44cf8be18482c2599b26c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/176fbd0e93a44cf8be18482c2599b26c 2024-11-11T06:40:11,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/176fbd0e93a44cf8be18482c2599b26c, entries=150, sequenceid=493, filesize=12.0 K 2024-11-11T06:40:11,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e 2024-11-11T06:40:11,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e, entries=150, sequenceid=493, filesize=12.0 K 2024-11-11T06:40:11,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/5665d04f11f042acae2fbaf02eb89814 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5665d04f11f042acae2fbaf02eb89814 2024-11-11T06:40:11,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5665d04f11f042acae2fbaf02eb89814, entries=150, sequenceid=493, filesize=12.0 K 2024-11-11T06:40:11,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=6.71 KB/6870 for 1b6d451b960d07000ebc207f625528a7 in 1290ms, sequenceid=493, compaction requested=true 2024-11-11T06:40:11,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:11,531 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:11,531 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b6d451b960d07000ebc207f625528a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:11,532 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:11,532 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:11,532 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/A is initiating minor compaction (all files) 2024-11-11T06:40:11,532 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/B is initiating minor compaction (all files) 2024-11-11T06:40:11,532 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/A in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:11,532 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/B in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:11,532 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/81667508dd9f478990c88319964f2b2f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/21e842f285f74120b77c69f84c34ea0d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/176fbd0e93a44cf8be18482c2599b26c] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=37.1 K 2024-11-11T06:40:11,532 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/50d26d29951a40a897433f1a53a6871e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8be9d578e0424e4ba4c600f08c734ba0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=37.1 K 2024-11-11T06:40:11,533 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81667508dd9f478990c88319964f2b2f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731307208974 2024-11-11T06:40:11,533 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d26d29951a40a897433f1a53a6871e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731307208974 2024-11-11T06:40:11,533 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e842f285f74120b77c69f84c34ea0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731307209934 2024-11-11T06:40:11,533 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 8be9d578e0424e4ba4c600f08c734ba0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731307209934 2024-11-11T06:40:11,534 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 176fbd0e93a44cf8be18482c2599b26c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731307210239 2024-11-11T06:40:11,534 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ab3b5fd5fcc4d73bbe02e238e7ef95e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731307210239 2024-11-11T06:40:11,546 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:11,546 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#A#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:11,547 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/53c4e8ce3e864f1697a935b171eb5e35 is 50, key is test_row_0/A:col10/1731307210239/Put/seqid=0 2024-11-11T06:40:11,547 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/6c0e4840b6804b938f35920d3661105d is 50, key is test_row_0/B:col10/1731307210239/Put/seqid=0 2024-11-11T06:40:11,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741939_1115 (size=13459) 2024-11-11T06:40:11,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741940_1116 (size=13459) 2024-11-11T06:40:11,565 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/53c4e8ce3e864f1697a935b171eb5e35 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/53c4e8ce3e864f1697a935b171eb5e35 2024-11-11T06:40:11,572 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/A of 1b6d451b960d07000ebc207f625528a7 into 53c4e8ce3e864f1697a935b171eb5e35(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:11,572 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:11,572 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/A, priority=13, startTime=1731307211531; duration=0sec 2024-11-11T06:40:11,572 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:11,572 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:A 2024-11-11T06:40:11,572 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:11,573 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:11,574 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1b6d451b960d07000ebc207f625528a7/C is initiating minor compaction (all files) 2024-11-11T06:40:11,574 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b6d451b960d07000ebc207f625528a7/C in TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:11,574 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/489d9f7796ea48a49785854c8d3f33ea, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/4c1fa36b6a7d4730a66e51e6627c4811, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5665d04f11f042acae2fbaf02eb89814] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp, totalSize=37.1 K 2024-11-11T06:40:11,574 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 489d9f7796ea48a49785854c8d3f33ea, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731307208974 2024-11-11T06:40:11,574 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c1fa36b6a7d4730a66e51e6627c4811, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731307209934 2024-11-11T06:40:11,575 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5665d04f11f042acae2fbaf02eb89814, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731307210239 2024-11-11T06:40:11,582 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b6d451b960d07000ebc207f625528a7#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:11,583 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/902e826e1b30426db043eb2dcf4abb21 is 50, key is test_row_0/C:col10/1731307210239/Put/seqid=0 2024-11-11T06:40:11,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741941_1117 (size=13459) 2024-11-11T06:40:11,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T06:40:11,957 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/6c0e4840b6804b938f35920d3661105d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/6c0e4840b6804b938f35920d3661105d 2024-11-11T06:40:11,962 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/B of 1b6d451b960d07000ebc207f625528a7 into 6c0e4840b6804b938f35920d3661105d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:11,962 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:11,963 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/B, priority=13, startTime=1731307211531; duration=0sec 2024-11-11T06:40:11,963 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:11,963 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:B 2024-11-11T06:40:11,994 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/902e826e1b30426db043eb2dcf4abb21 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/902e826e1b30426db043eb2dcf4abb21 2024-11-11T06:40:12,001 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b6d451b960d07000ebc207f625528a7/C of 1b6d451b960d07000ebc207f625528a7 into 902e826e1b30426db043eb2dcf4abb21(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:12,001 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b6d451b960d07000ebc207f625528a7: 2024-11-11T06:40:12,001 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., storeName=1b6d451b960d07000ebc207f625528a7/C, priority=13, startTime=1731307211531; duration=0sec 2024-11-11T06:40:12,001 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:12,001 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b6d451b960d07000ebc207f625528a7:C 2024-11-11T06:40:19,962 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T06:40:19,962 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:19,963 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:19,963 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:19,963 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:19,963 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T06:40:19,963 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-11T06:40:19,963 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-11T06:40:19,963 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3116 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3190 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1466 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4394 rows 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1470 2024-11-11T06:40:19,964 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4409 rows 2024-11-11T06:40:19,964 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T06:40:19,965 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:40:19,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:19,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:19,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:19,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T06:40:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T06:40:19,977 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307219977"}]},"ts":"1731307219977"} 2024-11-11T06:40:19,979 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T06:40:19,979 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T06:40:19,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:40:19,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, UNASSIGN}] 2024-11-11T06:40:19,987 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, UNASSIGN 2024-11-11T06:40:19,989 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1b6d451b960d07000ebc207f625528a7, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:19,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, UNASSIGN because future has completed 2024-11-11T06:40:19,992 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:40:19,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:40:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T06:40:20,151 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:20,151 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:40:20,152 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 1b6d451b960d07000ebc207f625528a7, disabling compactions & flushes 2024-11-11T06:40:20,152 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:20,152 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:20,153 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. after waiting 0 ms 2024-11-11T06:40:20,153 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:20,153 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 1b6d451b960d07000ebc207f625528a7 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-11T06:40:20,161 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/37afbfdc282d417db0a916a47fea7eda is 50, key is test_row_1/A:col10/1731307211140/Put/seqid=0 2024-11-11T06:40:20,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741942_1118 (size=9857) 2024-11-11T06:40:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T06:40:20,566 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/37afbfdc282d417db0a916a47fea7eda 2024-11-11T06:40:20,575 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/938b2095580c482692ec5cb51f8399a5 is 50, key is test_row_1/B:col10/1731307211140/Put/seqid=0 2024-11-11T06:40:20,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741943_1119 (size=9857) 2024-11-11T06:40:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T06:40:20,981 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/938b2095580c482692ec5cb51f8399a5 2024-11-11T06:40:20,988 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/f5f719c5d2c5423db9f7b49cd2407342 is 50, key is test_row_1/C:col10/1731307211140/Put/seqid=0 2024-11-11T06:40:20,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741944_1120 (size=9857) 2024-11-11T06:40:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T06:40:21,393 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/f5f719c5d2c5423db9f7b49cd2407342 2024-11-11T06:40:21,401 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/A/37afbfdc282d417db0a916a47fea7eda as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/37afbfdc282d417db0a916a47fea7eda 2024-11-11T06:40:21,406 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/37afbfdc282d417db0a916a47fea7eda, entries=100, sequenceid=501, filesize=9.6 K 2024-11-11T06:40:21,407 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/B/938b2095580c482692ec5cb51f8399a5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/938b2095580c482692ec5cb51f8399a5 2024-11-11T06:40:21,412 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/938b2095580c482692ec5cb51f8399a5, entries=100, sequenceid=501, filesize=9.6 K 2024-11-11T06:40:21,413 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/.tmp/C/f5f719c5d2c5423db9f7b49cd2407342 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f5f719c5d2c5423db9f7b49cd2407342 2024-11-11T06:40:21,417 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f5f719c5d2c5423db9f7b49cd2407342, entries=100, sequenceid=501, filesize=9.6 K 2024-11-11T06:40:21,418 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1b6d451b960d07000ebc207f625528a7 in 1264ms, sequenceid=501, compaction requested=false 2024-11-11T06:40:21,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/0f30091c3b1a4c11a61701a5400e4af2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dd9f12c23d954818b77994753e18d743, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f26978258a0240a091f42cf7478dc388, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c5a8a83d19a14a0eb0fd7cdc399608be, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/6a6fa7779d7741f8960f5ca4f2e8c8b8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/3e7533b40f9c4641b8890e18caa62d66, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c18171fa80d04603a8a788d320220bb5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8067c5aeec6045cc9e5fc040c942cd6e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a9a879a7a1be41259daa8265b3f690d0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/2a62ce536c8f44d596b461adabcb22c5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dff6b035b8004e73aed2cf6f5e118c9b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a427c3c51a5c4d2cb74079ef584d4584, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/054bdaea7514434aa94591ef70d43f6a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bde5d31b5c804f689f0c2045810218df, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/fec66bb1fd8649c98ed10a8a1b4f36fd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/9e0a5e55a501428f9ead8fa2d8d23876, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/783c6f85768e4d80884a7ed73f9199a5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7570fab4fc0247f7a0a30e6846d9e243, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/cd020605f9b846668013bcae2cb7079e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bf9131e5eba84580ba1a9db843716d2f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8ff734cb17f04f4da29f0c1182610788, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f4bff606eaea4c53a9d7fa7858ce4f8e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/af49bf8f9f6e4d0a9da0ff9c06599000, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/721e10a79b764609b05a7ebfdace78ba, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/00cad3ed35a5453fa12ced83f70e4419, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/e152aa77f8e84ed0976885c0f2553fa4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dfa25a6d2f53426392ee45f7b3251a1a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/ea0d52ce28ad4ccca66804d5278faf72, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a294449713384808ba511c2e6dcf81f5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7de5137649de437994f85a881b0cdaa0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/75a7030a6911492b86b167ff332df35e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/81667508dd9f478990c88319964f2b2f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/21e842f285f74120b77c69f84c34ea0d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/176fbd0e93a44cf8be18482c2599b26c] to archive 2024-11-11T06:40:21,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:40:21,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/0f30091c3b1a4c11a61701a5400e4af2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/0f30091c3b1a4c11a61701a5400e4af2 2024-11-11T06:40:21,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dd9f12c23d954818b77994753e18d743 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dd9f12c23d954818b77994753e18d743 2024-11-11T06:40:21,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f26978258a0240a091f42cf7478dc388 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f26978258a0240a091f42cf7478dc388 2024-11-11T06:40:21,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c5a8a83d19a14a0eb0fd7cdc399608be to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c5a8a83d19a14a0eb0fd7cdc399608be 2024-11-11T06:40:21,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/6a6fa7779d7741f8960f5ca4f2e8c8b8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/6a6fa7779d7741f8960f5ca4f2e8c8b8 2024-11-11T06:40:21,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/3e7533b40f9c4641b8890e18caa62d66 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/3e7533b40f9c4641b8890e18caa62d66 2024-11-11T06:40:21,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c18171fa80d04603a8a788d320220bb5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/c18171fa80d04603a8a788d320220bb5 2024-11-11T06:40:21,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8067c5aeec6045cc9e5fc040c942cd6e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8067c5aeec6045cc9e5fc040c942cd6e 2024-11-11T06:40:21,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a9a879a7a1be41259daa8265b3f690d0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a9a879a7a1be41259daa8265b3f690d0 2024-11-11T06:40:21,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/2a62ce536c8f44d596b461adabcb22c5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/2a62ce536c8f44d596b461adabcb22c5 2024-11-11T06:40:21,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dff6b035b8004e73aed2cf6f5e118c9b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dff6b035b8004e73aed2cf6f5e118c9b 2024-11-11T06:40:21,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a427c3c51a5c4d2cb74079ef584d4584 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a427c3c51a5c4d2cb74079ef584d4584 2024-11-11T06:40:21,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/054bdaea7514434aa94591ef70d43f6a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/054bdaea7514434aa94591ef70d43f6a 2024-11-11T06:40:21,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bde5d31b5c804f689f0c2045810218df to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bde5d31b5c804f689f0c2045810218df 2024-11-11T06:40:21,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/fec66bb1fd8649c98ed10a8a1b4f36fd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/fec66bb1fd8649c98ed10a8a1b4f36fd 2024-11-11T06:40:21,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/9e0a5e55a501428f9ead8fa2d8d23876 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/9e0a5e55a501428f9ead8fa2d8d23876 2024-11-11T06:40:21,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/783c6f85768e4d80884a7ed73f9199a5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/783c6f85768e4d80884a7ed73f9199a5 2024-11-11T06:40:21,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7570fab4fc0247f7a0a30e6846d9e243 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7570fab4fc0247f7a0a30e6846d9e243 2024-11-11T06:40:21,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/cd020605f9b846668013bcae2cb7079e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/cd020605f9b846668013bcae2cb7079e 2024-11-11T06:40:21,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bf9131e5eba84580ba1a9db843716d2f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/bf9131e5eba84580ba1a9db843716d2f 2024-11-11T06:40:21,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8ff734cb17f04f4da29f0c1182610788 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/8ff734cb17f04f4da29f0c1182610788 2024-11-11T06:40:21,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f4bff606eaea4c53a9d7fa7858ce4f8e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/f4bff606eaea4c53a9d7fa7858ce4f8e 2024-11-11T06:40:21,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/af49bf8f9f6e4d0a9da0ff9c06599000 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/af49bf8f9f6e4d0a9da0ff9c06599000 2024-11-11T06:40:21,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/721e10a79b764609b05a7ebfdace78ba to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/721e10a79b764609b05a7ebfdace78ba 2024-11-11T06:40:21,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/00cad3ed35a5453fa12ced83f70e4419 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/00cad3ed35a5453fa12ced83f70e4419 2024-11-11T06:40:21,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/e152aa77f8e84ed0976885c0f2553fa4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/e152aa77f8e84ed0976885c0f2553fa4 2024-11-11T06:40:21,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dfa25a6d2f53426392ee45f7b3251a1a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/dfa25a6d2f53426392ee45f7b3251a1a 2024-11-11T06:40:21,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/ea0d52ce28ad4ccca66804d5278faf72 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/ea0d52ce28ad4ccca66804d5278faf72 2024-11-11T06:40:21,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a294449713384808ba511c2e6dcf81f5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/a294449713384808ba511c2e6dcf81f5 2024-11-11T06:40:21,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7de5137649de437994f85a881b0cdaa0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/7de5137649de437994f85a881b0cdaa0 2024-11-11T06:40:21,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/75a7030a6911492b86b167ff332df35e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/75a7030a6911492b86b167ff332df35e 2024-11-11T06:40:21,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/81667508dd9f478990c88319964f2b2f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/81667508dd9f478990c88319964f2b2f 2024-11-11T06:40:21,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/21e842f285f74120b77c69f84c34ea0d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/21e842f285f74120b77c69f84c34ea0d 2024-11-11T06:40:21,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/176fbd0e93a44cf8be18482c2599b26c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/176fbd0e93a44cf8be18482c2599b26c 2024-11-11T06:40:21,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2eeabead1ac346a481255bb7f3f96525, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/18340bc4030543d180c54e059374b882, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/987a4a0989ce4f2184437d30c308a146, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/44f59547ef3349bdbfc5a4fbccb76bd4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7d499d4a8f40440e85280a88a09901c2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9bcc50bb59264fbeaa787bf4dd2fdc03, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/39ed3b3534a5429ea0524253cd55ee39, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/272035b8b4f0428c8bb75c1441ddbd1b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2c25c5dda0574b848aaf1f96b34ee737, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7599c8dbc6bf4a6691fc14807991270c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/92b19d2e35b24131b933cb7fa4cd5751, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/24984dc7c16d41ae8d4d0efe8422445d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c33beecf5aba4ab28292f1554096a87d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8cd100dd4f034e9b879ba75b9fda92aa, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/eb6f28ef94514e6682777cbd5635f15d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4ea25e35d66b46f59fe1bb994e23033d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2fa4c86fca5644d0b06ec595ccb0a378, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d37e3ed888b347839831f8f95c9367fc, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c4eb0ec8ff8e49c39067ed2493d2a5cf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/a21d4260dc84405db69dc926c29d3e1f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d84b766389e1403387633189c3be9c52, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/1425c5daa10a41bea90c7ab339510f2b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4fbaddf035cf43df94d127b5d9296177, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f0cf597fd5cd4d879ec3403e21151dba, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f414109995044e3e8144d5b5a35a4c58, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/246a25f6428547a0a1b94518fbeba32e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/5bf0551b559b4ed4ac59ba0c03f227fd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c8f43ea3efdd4685b094fddbabe0e7a7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/920de46362dc4bfd8dd4c70bc24fa529, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/50d26d29951a40a897433f1a53a6871e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9529b9a50f094980be4de0e4e4241364, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8be9d578e0424e4ba4c600f08c734ba0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e] to archive 2024-11-11T06:40:21,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:40:21,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2eeabead1ac346a481255bb7f3f96525 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2eeabead1ac346a481255bb7f3f96525 2024-11-11T06:40:21,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/18340bc4030543d180c54e059374b882 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/18340bc4030543d180c54e059374b882 2024-11-11T06:40:21,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/987a4a0989ce4f2184437d30c308a146 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/987a4a0989ce4f2184437d30c308a146 2024-11-11T06:40:21,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/44f59547ef3349bdbfc5a4fbccb76bd4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/44f59547ef3349bdbfc5a4fbccb76bd4 2024-11-11T06:40:21,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7d499d4a8f40440e85280a88a09901c2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7d499d4a8f40440e85280a88a09901c2 2024-11-11T06:40:21,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9bcc50bb59264fbeaa787bf4dd2fdc03 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9bcc50bb59264fbeaa787bf4dd2fdc03 2024-11-11T06:40:21,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/39ed3b3534a5429ea0524253cd55ee39 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/39ed3b3534a5429ea0524253cd55ee39 2024-11-11T06:40:21,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/272035b8b4f0428c8bb75c1441ddbd1b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/272035b8b4f0428c8bb75c1441ddbd1b 2024-11-11T06:40:21,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2c25c5dda0574b848aaf1f96b34ee737 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2c25c5dda0574b848aaf1f96b34ee737 2024-11-11T06:40:21,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7599c8dbc6bf4a6691fc14807991270c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/7599c8dbc6bf4a6691fc14807991270c 2024-11-11T06:40:21,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/92b19d2e35b24131b933cb7fa4cd5751 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/92b19d2e35b24131b933cb7fa4cd5751 2024-11-11T06:40:21,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/24984dc7c16d41ae8d4d0efe8422445d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/24984dc7c16d41ae8d4d0efe8422445d 2024-11-11T06:40:21,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c33beecf5aba4ab28292f1554096a87d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c33beecf5aba4ab28292f1554096a87d 2024-11-11T06:40:21,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8cd100dd4f034e9b879ba75b9fda92aa to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8cd100dd4f034e9b879ba75b9fda92aa 2024-11-11T06:40:21,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/eb6f28ef94514e6682777cbd5635f15d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/eb6f28ef94514e6682777cbd5635f15d 2024-11-11T06:40:21,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4ea25e35d66b46f59fe1bb994e23033d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4ea25e35d66b46f59fe1bb994e23033d 2024-11-11T06:40:21,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c3dbb9cdcb21465dbcfdee7ffbe3d8c1 2024-11-11T06:40:21,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2fa4c86fca5644d0b06ec595ccb0a378 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2fa4c86fca5644d0b06ec595ccb0a378 2024-11-11T06:40:21,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d37e3ed888b347839831f8f95c9367fc to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d37e3ed888b347839831f8f95c9367fc 2024-11-11T06:40:21,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c4eb0ec8ff8e49c39067ed2493d2a5cf to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c4eb0ec8ff8e49c39067ed2493d2a5cf 2024-11-11T06:40:21,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/a21d4260dc84405db69dc926c29d3e1f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/a21d4260dc84405db69dc926c29d3e1f 2024-11-11T06:40:21,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d84b766389e1403387633189c3be9c52 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/d84b766389e1403387633189c3be9c52 2024-11-11T06:40:21,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/1425c5daa10a41bea90c7ab339510f2b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/1425c5daa10a41bea90c7ab339510f2b 2024-11-11T06:40:21,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4fbaddf035cf43df94d127b5d9296177 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/4fbaddf035cf43df94d127b5d9296177 2024-11-11T06:40:21,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f0cf597fd5cd4d879ec3403e21151dba to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f0cf597fd5cd4d879ec3403e21151dba 2024-11-11T06:40:21,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f414109995044e3e8144d5b5a35a4c58 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/f414109995044e3e8144d5b5a35a4c58 2024-11-11T06:40:21,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/246a25f6428547a0a1b94518fbeba32e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/246a25f6428547a0a1b94518fbeba32e 2024-11-11T06:40:21,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/5bf0551b559b4ed4ac59ba0c03f227fd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/5bf0551b559b4ed4ac59ba0c03f227fd 2024-11-11T06:40:21,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c8f43ea3efdd4685b094fddbabe0e7a7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/c8f43ea3efdd4685b094fddbabe0e7a7 2024-11-11T06:40:21,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/920de46362dc4bfd8dd4c70bc24fa529 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/920de46362dc4bfd8dd4c70bc24fa529 2024-11-11T06:40:21,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/50d26d29951a40a897433f1a53a6871e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/50d26d29951a40a897433f1a53a6871e 2024-11-11T06:40:21,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9529b9a50f094980be4de0e4e4241364 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/9529b9a50f094980be4de0e4e4241364 2024-11-11T06:40:21,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8be9d578e0424e4ba4c600f08c734ba0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/8be9d578e0424e4ba4c600f08c734ba0 2024-11-11T06:40:21,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/2ab3b5fd5fcc4d73bbe02e238e7ef95e 2024-11-11T06:40:21,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/d9624ffb013c4aa2aa812e3868e78619, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/abe7459cb5f24b81ac73bd8f7761bb0e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f3aa4238553b4c2bba6c2ba8858a6564, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/24f76114a5e2470a883910dc4cdeb250, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b9533b08d4eb45299d891462aece5678, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2f9e8698a197481992ef6d8684e6ec0e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/104ac9bc5a2b4623b73b8d7408b0dddd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3818a52bf1be4e5fb19b3549819d694c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3509c252bb7f482eae7be51d855aba22, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2564c9972ca144d9aad0d3cd27f3b980, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cb550a1b1daf455a9473db207f2fbeec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/ce6ce5e6194e4bd6bc81f0509e292721, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b0adad4d66a342528fdc99ab98ba2b54, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b14998df7158440f86d4134d182806cd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/dbdb1b9693ee44218891424e64d64ea1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fcf07021a5eb4de9a793d2ab2c288ea7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3bb20259b3334b71833c4507b3cab416, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5a975ce8b4b44d6facf0a990dd4f0cc1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/9b5e373339e64265995e8a5271d2753e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/57b03b22d50445eaa6fc53895622ccba, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3e259b2701c04983b42881f1b1479f4e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/795f219922304a989362eff03be6142d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cc6307b40cbf4fbd967a4470d253d579, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fb7a60c5d297492b9211c31cdbccef43, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/aa24a14473a141349df3cc3997559837, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/6dd8b6159c8a4a4897c479b564795d01, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cbd3edff73c348baa766eee337665ade, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/56b7b17ccfc044858cb9600f973bc9ad, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/0147cccac6cd4ebaa3dad444a97c0690, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/be040aff8b16443ab299b5b4cd73c892, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/489d9f7796ea48a49785854c8d3f33ea, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/1429c01d5b2543d3a6e3cb127ef0adc6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/4c1fa36b6a7d4730a66e51e6627c4811, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5665d04f11f042acae2fbaf02eb89814] to archive 2024-11-11T06:40:21,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:40:21,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/d9624ffb013c4aa2aa812e3868e78619 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/d9624ffb013c4aa2aa812e3868e78619 2024-11-11T06:40:21,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/abe7459cb5f24b81ac73bd8f7761bb0e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/abe7459cb5f24b81ac73bd8f7761bb0e 2024-11-11T06:40:21,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f3aa4238553b4c2bba6c2ba8858a6564 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f3aa4238553b4c2bba6c2ba8858a6564 2024-11-11T06:40:21,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/24f76114a5e2470a883910dc4cdeb250 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/24f76114a5e2470a883910dc4cdeb250 2024-11-11T06:40:21,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b9533b08d4eb45299d891462aece5678 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b9533b08d4eb45299d891462aece5678 2024-11-11T06:40:21,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2f9e8698a197481992ef6d8684e6ec0e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2f9e8698a197481992ef6d8684e6ec0e 2024-11-11T06:40:21,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/104ac9bc5a2b4623b73b8d7408b0dddd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/104ac9bc5a2b4623b73b8d7408b0dddd 2024-11-11T06:40:21,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3818a52bf1be4e5fb19b3549819d694c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3818a52bf1be4e5fb19b3549819d694c 2024-11-11T06:40:21,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3509c252bb7f482eae7be51d855aba22 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3509c252bb7f482eae7be51d855aba22 2024-11-11T06:40:21,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2564c9972ca144d9aad0d3cd27f3b980 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/2564c9972ca144d9aad0d3cd27f3b980 2024-11-11T06:40:21,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cb550a1b1daf455a9473db207f2fbeec to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cb550a1b1daf455a9473db207f2fbeec 2024-11-11T06:40:21,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/ce6ce5e6194e4bd6bc81f0509e292721 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/ce6ce5e6194e4bd6bc81f0509e292721 2024-11-11T06:40:21,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b0adad4d66a342528fdc99ab98ba2b54 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b0adad4d66a342528fdc99ab98ba2b54 2024-11-11T06:40:21,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b14998df7158440f86d4134d182806cd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/b14998df7158440f86d4134d182806cd 2024-11-11T06:40:21,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/dbdb1b9693ee44218891424e64d64ea1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/dbdb1b9693ee44218891424e64d64ea1 2024-11-11T06:40:21,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fcf07021a5eb4de9a793d2ab2c288ea7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fcf07021a5eb4de9a793d2ab2c288ea7 2024-11-11T06:40:21,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3bb20259b3334b71833c4507b3cab416 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3bb20259b3334b71833c4507b3cab416 2024-11-11T06:40:21,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5a975ce8b4b44d6facf0a990dd4f0cc1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5a975ce8b4b44d6facf0a990dd4f0cc1 2024-11-11T06:40:21,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/9b5e373339e64265995e8a5271d2753e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/9b5e373339e64265995e8a5271d2753e 2024-11-11T06:40:21,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/57b03b22d50445eaa6fc53895622ccba to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/57b03b22d50445eaa6fc53895622ccba 2024-11-11T06:40:21,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3e259b2701c04983b42881f1b1479f4e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/3e259b2701c04983b42881f1b1479f4e 2024-11-11T06:40:21,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/795f219922304a989362eff03be6142d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/795f219922304a989362eff03be6142d 2024-11-11T06:40:21,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cc6307b40cbf4fbd967a4470d253d579 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cc6307b40cbf4fbd967a4470d253d579 2024-11-11T06:40:21,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fb7a60c5d297492b9211c31cdbccef43 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/fb7a60c5d297492b9211c31cdbccef43 2024-11-11T06:40:21,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/aa24a14473a141349df3cc3997559837 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/aa24a14473a141349df3cc3997559837 2024-11-11T06:40:21,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/6dd8b6159c8a4a4897c479b564795d01 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/6dd8b6159c8a4a4897c479b564795d01 2024-11-11T06:40:21,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cbd3edff73c348baa766eee337665ade to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/cbd3edff73c348baa766eee337665ade 2024-11-11T06:40:21,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/56b7b17ccfc044858cb9600f973bc9ad to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/56b7b17ccfc044858cb9600f973bc9ad 2024-11-11T06:40:21,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/0147cccac6cd4ebaa3dad444a97c0690 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/0147cccac6cd4ebaa3dad444a97c0690 2024-11-11T06:40:21,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/be040aff8b16443ab299b5b4cd73c892 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/be040aff8b16443ab299b5b4cd73c892 2024-11-11T06:40:21,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/489d9f7796ea48a49785854c8d3f33ea to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/489d9f7796ea48a49785854c8d3f33ea 2024-11-11T06:40:21,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/1429c01d5b2543d3a6e3cb127ef0adc6 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/1429c01d5b2543d3a6e3cb127ef0adc6 2024-11-11T06:40:21,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/4c1fa36b6a7d4730a66e51e6627c4811 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/4c1fa36b6a7d4730a66e51e6627c4811 2024-11-11T06:40:21,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5665d04f11f042acae2fbaf02eb89814 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/5665d04f11f042acae2fbaf02eb89814 2024-11-11T06:40:21,582 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/recovered.edits/504.seqid, newMaxSeqId=504, maxSeqId=1 2024-11-11T06:40:21,583 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. 2024-11-11T06:40:21,583 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 1b6d451b960d07000ebc207f625528a7: Waiting for close lock at 1731307220152Running coprocessor pre-close hooks at 1731307220152Disabling compacts and flushes for region at 1731307220152Disabling writes for close at 1731307220153 (+1 ms)Obtaining lock to block concurrent updates at 1731307220153Preparing flush snapshotting stores in 1b6d451b960d07000ebc207f625528a7 at 1731307220153Finished memstore snapshotting TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731307220153Flushing stores of TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7. at 1731307220154 (+1 ms)Flushing 1b6d451b960d07000ebc207f625528a7/A: creating writer at 1731307220154Flushing 1b6d451b960d07000ebc207f625528a7/A: appending metadata at 1731307220160 (+6 ms)Flushing 1b6d451b960d07000ebc207f625528a7/A: closing flushed file at 1731307220160Flushing 1b6d451b960d07000ebc207f625528a7/B: creating writer at 1731307220571 (+411 ms)Flushing 1b6d451b960d07000ebc207f625528a7/B: appending metadata at 1731307220574 (+3 ms)Flushing 1b6d451b960d07000ebc207f625528a7/B: closing flushed file at 1731307220575 (+1 ms)Flushing 1b6d451b960d07000ebc207f625528a7/C: creating writer at 1731307220985 (+410 ms)Flushing 1b6d451b960d07000ebc207f625528a7/C: appending metadata at 1731307220988 (+3 ms)Flushing 1b6d451b960d07000ebc207f625528a7/C: closing flushed file at 1731307220988Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d49c4a1: reopening flushed file at 1731307221400 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15166245: reopening flushed file at 1731307221406 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a116260: reopening flushed file at 1731307221412 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1b6d451b960d07000ebc207f625528a7 in 1264ms, sequenceid=501, compaction requested=false at 1731307221418 (+6 ms)Writing region close event to WAL at 1731307221579 (+161 ms)Running coprocessor post-close hooks at 1731307221583 (+4 ms)Closed at 1731307221583 2024-11-11T06:40:21,585 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:21,586 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=1b6d451b960d07000ebc207f625528a7, regionState=CLOSED 2024-11-11T06:40:21,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:40:21,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T06:40:21,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 1b6d451b960d07000ebc207f625528a7, server=ce2cf0244921,35807,1731307183669 in 1.5970 sec 2024-11-11T06:40:21,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-11T06:40:21,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1b6d451b960d07000ebc207f625528a7, UNASSIGN in 1.6050 sec 2024-11-11T06:40:21,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T06:40:21,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6120 sec 2024-11-11T06:40:21,596 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307221596"}]},"ts":"1731307221596"} 2024-11-11T06:40:21,598 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T06:40:21,598 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T06:40:21,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6260 sec 2024-11-11T06:40:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T06:40:22,129 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:22,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T06:40:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:22,138 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T06:40:22,141 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:22,146 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:22,150 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/recovered.edits] 2024-11-11T06:40:22,154 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/37afbfdc282d417db0a916a47fea7eda to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/37afbfdc282d417db0a916a47fea7eda 2024-11-11T06:40:22,155 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/53c4e8ce3e864f1697a935b171eb5e35 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/A/53c4e8ce3e864f1697a935b171eb5e35 2024-11-11T06:40:22,157 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/6c0e4840b6804b938f35920d3661105d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/6c0e4840b6804b938f35920d3661105d 2024-11-11T06:40:22,158 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/938b2095580c482692ec5cb51f8399a5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/B/938b2095580c482692ec5cb51f8399a5 2024-11-11T06:40:22,160 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/902e826e1b30426db043eb2dcf4abb21 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/902e826e1b30426db043eb2dcf4abb21 2024-11-11T06:40:22,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f5f719c5d2c5423db9f7b49cd2407342 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/C/f5f719c5d2c5423db9f7b49cd2407342 2024-11-11T06:40:22,164 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/recovered.edits/504.seqid to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7/recovered.edits/504.seqid 2024-11-11T06:40:22,164 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1b6d451b960d07000ebc207f625528a7 2024-11-11T06:40:22,164 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T06:40:22,169 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:22,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T06:40:22,177 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T06:40:22,204 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T06:40:22,205 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:22,205 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T06:40:22,205 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731307222205"}]},"ts":"9223372036854775807"} 2024-11-11T06:40:22,209 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T06:40:22,209 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1b6d451b960d07000ebc207f625528a7, NAME => 'TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T06:40:22,210 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T06:40:22,210 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731307222210"}]},"ts":"9223372036854775807"} 2024-11-11T06:40:22,213 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T06:40:22,214 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:22,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-11T06:40:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T06:40:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T06:40:22,250 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:22,264 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ce2cf0244921:35807-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_448787011_22 at /127.0.0.1:43224 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 319) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11710 (was 12158) 2024-11-11T06:40:22,272 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=427, ProcessCount=11, AvailableMemoryMB=11709 2024-11-11T06:40:22,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:40:22,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:40:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:22,276 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T06:40:22,276 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:22,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-11T06:40:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T06:40:22,277 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T06:40:22,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741945_1121 (size=959) 2024-11-11T06:40:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T06:40:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T06:40:22,685 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 2024-11-11T06:40:22,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741946_1122 (size=53) 2024-11-11T06:40:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T06:40:23,097 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:40:23,098 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1afd384ae6d17851e72f641429f124f2, disabling compactions & flushes 2024-11-11T06:40:23,098 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:23,098 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:23,098 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. after waiting 0 ms 2024-11-11T06:40:23,098 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:23,098 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:23,098 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1afd384ae6d17851e72f641429f124f2: Waiting for close lock at 1731307223098Disabling compacts and flushes for region at 1731307223098Disabling writes for close at 1731307223098Writing region close event to WAL at 1731307223098Closed at 1731307223098 2024-11-11T06:40:23,100 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T06:40:23,101 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731307223100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731307223100"}]},"ts":"1731307223100"} 2024-11-11T06:40:23,105 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T06:40:23,107 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T06:40:23,107 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307223107"}]},"ts":"1731307223107"} 2024-11-11T06:40:23,109 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T06:40:23,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, ASSIGN}] 2024-11-11T06:40:23,111 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, ASSIGN 2024-11-11T06:40:23,112 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, ASSIGN; state=OFFLINE, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=false 2024-11-11T06:40:23,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T06:40:23,262 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1afd384ae6d17851e72f641429f124f2, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:23,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, ASSIGN because future has completed 2024-11-11T06:40:23,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:40:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T06:40:23,424 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:23,425 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:40:23,425 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,425 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:40:23,426 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,426 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,428 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,431 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1afd384ae6d17851e72f641429f124f2 columnFamilyName A 2024-11-11T06:40:23,432 DEBUG [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:23,432 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(327): Store=1afd384ae6d17851e72f641429f124f2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:23,432 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,435 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1afd384ae6d17851e72f641429f124f2 columnFamilyName B 2024-11-11T06:40:23,435 DEBUG [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:23,436 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(327): Store=1afd384ae6d17851e72f641429f124f2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:23,436 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,438 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1afd384ae6d17851e72f641429f124f2 columnFamilyName C 2024-11-11T06:40:23,438 DEBUG [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:23,438 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(327): Store=1afd384ae6d17851e72f641429f124f2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:23,439 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,439 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,440 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,441 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,441 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,442 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:40:23,443 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,445 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T06:40:23,446 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 1afd384ae6d17851e72f641429f124f2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62701580, jitterRate=-0.06567364931106567}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:40:23,446 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:23,447 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 1afd384ae6d17851e72f641429f124f2: Running coprocessor pre-open hook at 1731307223426Writing region info on filesystem at 1731307223426Initializing all the Stores at 1731307223427 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307223428 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307223428Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307223428Cleaning up temporary data from old regions at 1731307223441 (+13 ms)Running coprocessor post-open hooks at 1731307223446 (+5 ms)Region opened successfully at 1731307223447 (+1 ms) 2024-11-11T06:40:23,447 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., pid=30, masterSystemTime=1731307223418 2024-11-11T06:40:23,449 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:23,449 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:23,450 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1afd384ae6d17851e72f641429f124f2, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:23,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:40:23,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T06:40:23,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 in 188 msec 2024-11-11T06:40:23,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-11T06:40:23,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, ASSIGN in 345 msec 2024-11-11T06:40:23,458 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T06:40:23,458 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307223458"}]},"ts":"1731307223458"} 2024-11-11T06:40:23,460 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T06:40:23,460 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T06:40:23,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-11T06:40:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-11T06:40:24,420 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:24,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:24,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:24,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:24,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:24,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f1df996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:24,428 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79906bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:24,429 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:24,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:24,434 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T06:40:24,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,40845,1731307182847 2024-11-11T06:40:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16e27d07 2024-11-11T06:40:24,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T06:40:24,437 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T06:40:24,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:40:24,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:40:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:24,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741947_1123 (size=995) 2024-11-11T06:40:24,864 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T06:40:24,864 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T06:40:24,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:40:24,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, REOPEN/MOVE}] 2024-11-11T06:40:24,879 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, REOPEN/MOVE 2024-11-11T06:40:24,880 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1afd384ae6d17851e72f641429f124f2, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:24,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, REOPEN/MOVE because future has completed 2024-11-11T06:40:24,882 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:40:24,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:40:25,035 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,035 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:40:25,036 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 1afd384ae6d17851e72f641429f124f2, disabling compactions & flushes 2024-11-11T06:40:25,036 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,036 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,036 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. after waiting 0 ms 2024-11-11T06:40:25,036 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,045 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T06:40:25,046 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 1afd384ae6d17851e72f641429f124f2: Waiting for close lock at 1731307225036Running coprocessor pre-close hooks at 1731307225036Disabling compacts and flushes for region at 1731307225036Disabling writes for close at 1731307225036Writing region close event to WAL at 1731307225037 (+1 ms)Running coprocessor post-close hooks at 1731307225046 (+9 ms)Closed at 1731307225046 2024-11-11T06:40:25,046 WARN [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 1afd384ae6d17851e72f641429f124f2 to self. 2024-11-11T06:40:25,049 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,050 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1afd384ae6d17851e72f641429f124f2, regionState=CLOSED 2024-11-11T06:40:25,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:40:25,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T06:40:25,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 in 171 msec 2024-11-11T06:40:25,056 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, REOPEN/MOVE; state=CLOSED, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=true 2024-11-11T06:40:25,207 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1afd384ae6d17851e72f641429f124f2, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, REOPEN/MOVE because future has completed 2024-11-11T06:40:25,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:40:25,365 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,365 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:40:25,365 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,365 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:40:25,365 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,365 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,369 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,375 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1afd384ae6d17851e72f641429f124f2 columnFamilyName A 2024-11-11T06:40:25,378 DEBUG [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:25,378 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(327): Store=1afd384ae6d17851e72f641429f124f2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:25,379 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,380 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1afd384ae6d17851e72f641429f124f2 columnFamilyName B 2024-11-11T06:40:25,380 DEBUG [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:25,380 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(327): Store=1afd384ae6d17851e72f641429f124f2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:25,380 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,381 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1afd384ae6d17851e72f641429f124f2 columnFamilyName C 2024-11-11T06:40:25,381 DEBUG [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:25,382 INFO [StoreOpener-1afd384ae6d17851e72f641429f124f2-1 {}] regionserver.HStore(327): Store=1afd384ae6d17851e72f641429f124f2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:25,382 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,383 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,384 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,385 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,385 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,385 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:40:25,387 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,387 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1afd384ae6d17851e72f641429f124f2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70588594, jitterRate=0.051852017641067505}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:40:25,388 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,389 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1afd384ae6d17851e72f641429f124f2: Running coprocessor pre-open hook at 1731307225365Writing region info on filesystem at 1731307225365Initializing all the Stores at 1731307225366 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307225366Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307225367 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307225367Cleaning up temporary data from old regions at 1731307225385 (+18 ms)Running coprocessor post-open hooks at 1731307225388 (+3 ms)Region opened successfully at 1731307225389 (+1 ms) 2024-11-11T06:40:25,390 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., pid=35, masterSystemTime=1731307225361 2024-11-11T06:40:25,392 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,392 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,392 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=1afd384ae6d17851e72f641429f124f2, regionState=OPEN, openSeqNum=5, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:40:25,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-11T06:40:25,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 in 186 msec 2024-11-11T06:40:25,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-11T06:40:25,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, REOPEN/MOVE in 519 msec 2024-11-11T06:40:25,402 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T06:40:25,402 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-11T06:40:25,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-11-11T06:40:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T06:40:25,407 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:25,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e92a731, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3310dfa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,413 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be3d499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,415 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf52e05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,418 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d15b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,419 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f70ec80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,422 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a11042, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,423 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7b912f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,425 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec670b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,427 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2e6eaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,430 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5b6d36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c202834, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,433 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c72ce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,434 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6c58b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,436 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d6fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721efe8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:25,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:25,440 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce2cd12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:25,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:25,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:25,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c31ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:25,443 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,443 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,443 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,443 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,443 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,447 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,447 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,447 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,448 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-11T06:40:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T06:40:25,453 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,454 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:25,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,457 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,463 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:25,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:40:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:25,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:25,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:40:25,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a72079110be44992acdb9ba91793c5f2_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307225461/Put/seqid=0 2024-11-11T06:40:25,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307285504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307285505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307285507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307285508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307285512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T06:40:25,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741948_1124 (size=14594) 2024-11-11T06:40:25,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:25,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:25,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307285619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307285619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307285619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307285629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307285641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:25,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:25,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T06:40:25,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307285829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307285829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307285830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307285840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307285860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:25,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:25,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:25,973 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:25,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a72079110be44992acdb9ba91793c5f2_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a72079110be44992acdb9ba91793c5f2_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:25,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/3b5cce1a29884c13aaad4ba900f61e70, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:25,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/3b5cce1a29884c13aaad4ba900f61e70 is 175, key is test_row_0/A:col10/1731307225461/Put/seqid=0 2024-11-11T06:40:26,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741949_1125 (size=39549) 2024-11-11T06:40:26,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/3b5cce1a29884c13aaad4ba900f61e70 2024-11-11T06:40:26,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/5f1d108e6535437ea9f29757d7c103b5 is 50, key is test_row_0/B:col10/1731307225461/Put/seqid=0 2024-11-11T06:40:26,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:26,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:26,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T06:40:26,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741950_1126 (size=12001) 2024-11-11T06:40:26,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/5f1d108e6535437ea9f29757d7c103b5 2024-11-11T06:40:26,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307286139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/a4b4f228941e4ba7b36626cff87b9a42 is 50, key is test_row_0/C:col10/1731307225461/Put/seqid=0 2024-11-11T06:40:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307286139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307286139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741951_1127 (size=12001) 2024-11-11T06:40:26,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/a4b4f228941e4ba7b36626cff87b9a42 2024-11-11T06:40:26,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307286150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/3b5cce1a29884c13aaad4ba900f61e70 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/3b5cce1a29884c13aaad4ba900f61e70 2024-11-11T06:40:26,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/3b5cce1a29884c13aaad4ba900f61e70, entries=200, sequenceid=16, filesize=38.6 K 2024-11-11T06:40:26,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/5f1d108e6535437ea9f29757d7c103b5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5f1d108e6535437ea9f29757d7c103b5 2024-11-11T06:40:26,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5f1d108e6535437ea9f29757d7c103b5, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T06:40:26,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/a4b4f228941e4ba7b36626cff87b9a42 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/a4b4f228941e4ba7b36626cff87b9a42 2024-11-11T06:40:26,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/a4b4f228941e4ba7b36626cff87b9a42, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T06:40:26,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1afd384ae6d17851e72f641429f124f2 in 723ms, sequenceid=16, compaction requested=false 2024-11-11T06:40:26,186 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T06:40:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:26,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:26,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:40:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:26,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110f52f40dcbc453eac84c03130588440_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307225506/Put/seqid=0 2024-11-11T06:40:26,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741952_1128 (size=12154) 2024-11-11T06:40:26,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307286303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:26,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:26,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307286419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:26,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:26,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T06:40:26,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307286629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,647 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:26,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307286649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307286649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307286649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110f52f40dcbc453eac84c03130588440_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110f52f40dcbc453eac84c03130588440_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:26,659 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/a1bd722f2672475699fb2608b7b040a8, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:26,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/a1bd722f2672475699fb2608b7b040a8 is 175, key is test_row_0/A:col10/1731307225506/Put/seqid=0 2024-11-11T06:40:26,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307286659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741953_1129 (size=30955) 2024-11-11T06:40:26,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/a1bd722f2672475699fb2608b7b040a8 2024-11-11T06:40:26,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:26,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:26,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/838f78f627ee4deba9489481a85c5a12 is 50, key is test_row_0/B:col10/1731307225506/Put/seqid=0 2024-11-11T06:40:26,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741954_1130 (size=12001) 2024-11-11T06:40:26,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:26,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:26,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307286939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:26,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:26,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:26,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:27,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/838f78f627ee4deba9489481a85c5a12 2024-11-11T06:40:27,140 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T06:40:27,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:27,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:27,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:27,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:27,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:27,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6b072298643346d3b902e88c5fcc5122 is 50, key is test_row_0/C:col10/1731307225506/Put/seqid=0 2024-11-11T06:40:27,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741955_1131 (size=12001) 2024-11-11T06:40:27,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6b072298643346d3b902e88c5fcc5122 2024-11-11T06:40:27,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/a1bd722f2672475699fb2608b7b040a8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a1bd722f2672475699fb2608b7b040a8 2024-11-11T06:40:27,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a1bd722f2672475699fb2608b7b040a8, entries=150, sequenceid=41, filesize=30.2 K 2024-11-11T06:40:27,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/838f78f627ee4deba9489481a85c5a12 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/838f78f627ee4deba9489481a85c5a12 2024-11-11T06:40:27,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/838f78f627ee4deba9489481a85c5a12, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T06:40:27,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6b072298643346d3b902e88c5fcc5122 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6b072298643346d3b902e88c5fcc5122 2024-11-11T06:40:27,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6b072298643346d3b902e88c5fcc5122, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T06:40:27,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1afd384ae6d17851e72f641429f124f2 in 1070ms, sequenceid=41, compaction requested=false 2024-11-11T06:40:27,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:27,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T06:40:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:27,312 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:40:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111117a07caa417642599d1757da5e9dd0ab_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307226300/Put/seqid=0 2024-11-11T06:40:27,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741956_1132 (size=12154) 2024-11-11T06:40:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:27,404 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111117a07caa417642599d1757da5e9dd0ab_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117a07caa417642599d1757da5e9dd0ab_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:27,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/14d6958cf1ff48af95e4b170d21d8d6c, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/14d6958cf1ff48af95e4b170d21d8d6c is 175, key is test_row_0/A:col10/1731307226300/Put/seqid=0 2024-11-11T06:40:27,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:27,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741957_1133 (size=30955) 2024-11-11T06:40:27,459 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/14d6958cf1ff48af95e4b170d21d8d6c 2024-11-11T06:40:27,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/2fcc6d541b1f4fd1acea26f674145de2 is 50, key is test_row_0/B:col10/1731307226300/Put/seqid=0 2024-11-11T06:40:27,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741958_1134 (size=12001) 2024-11-11T06:40:27,592 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/2fcc6d541b1f4fd1acea26f674145de2 2024-11-11T06:40:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T06:40:27,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/de3c2b0ea3be4eacba4df55c28737b42 is 50, key is test_row_0/C:col10/1731307226300/Put/seqid=0 2024-11-11T06:40:27,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741959_1135 (size=12001) 2024-11-11T06:40:27,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307287671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307287672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:27,684 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1afd384ae6d17851e72f641429f124f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307287681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:27,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307287682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:27,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307287683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:27,782 INFO [master/ce2cf0244921:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T06:40:27,782 INFO [master/ce2cf0244921:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T06:40:27,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307287789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307287799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307287800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307287800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307288007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307288009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307288019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307288020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,072 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/de3c2b0ea3be4eacba4df55c28737b42 2024-11-11T06:40:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/14d6958cf1ff48af95e4b170d21d8d6c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/14d6958cf1ff48af95e4b170d21d8d6c 2024-11-11T06:40:28,087 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/14d6958cf1ff48af95e4b170d21d8d6c, entries=150, sequenceid=52, filesize=30.2 K 2024-11-11T06:40:28,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/2fcc6d541b1f4fd1acea26f674145de2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2fcc6d541b1f4fd1acea26f674145de2 2024-11-11T06:40:28,105 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2fcc6d541b1f4fd1acea26f674145de2, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T06:40:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/de3c2b0ea3be4eacba4df55c28737b42 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/de3c2b0ea3be4eacba4df55c28737b42 2024-11-11T06:40:28,112 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/de3c2b0ea3be4eacba4df55c28737b42, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T06:40:28,114 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1afd384ae6d17851e72f641429f124f2 in 802ms, sequenceid=52, compaction requested=true 2024-11-11T06:40:28,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:28,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:28,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-11T06:40:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-11T06:40:28,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-11T06:40:28,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6620 sec 2024-11-11T06:40:28,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.6720 sec 2024-11-11T06:40:28,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:28,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T06:40:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:28,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119d59ea676cbc49d6a64f2537a146f105_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307228322/Put/seqid=0 2024-11-11T06:40:28,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307288341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307288348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307288353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307288359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741960_1136 (size=14594) 2024-11-11T06:40:28,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:28,422 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119d59ea676cbc49d6a64f2537a146f105_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119d59ea676cbc49d6a64f2537a146f105_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:28,425 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/015b3ee06a804e7da13b5546ed8dcbb3, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:28,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/015b3ee06a804e7da13b5546ed8dcbb3 is 175, key is test_row_0/A:col10/1731307228322/Put/seqid=0 2024-11-11T06:40:28,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741961_1137 (size=39549) 2024-11-11T06:40:28,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307288471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307288472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307288471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307288679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307288689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307288689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307288859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:28,874 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/015b3ee06a804e7da13b5546ed8dcbb3 2024-11-11T06:40:28,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/4ec1cf1e010642d48841a0ed9fc6b4cc is 50, key is test_row_0/B:col10/1731307228322/Put/seqid=0 2024-11-11T06:40:28,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741962_1138 (size=12001) 2024-11-11T06:40:28,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/4ec1cf1e010642d48841a0ed9fc6b4cc 2024-11-11T06:40:28,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/b6f05480f0554bad9f4fb598ba10a4e2 is 50, key is test_row_0/C:col10/1731307228322/Put/seqid=0 2024-11-11T06:40:28,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741963_1139 (size=12001) 2024-11-11T06:40:28,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/b6f05480f0554bad9f4fb598ba10a4e2 2024-11-11T06:40:28,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/015b3ee06a804e7da13b5546ed8dcbb3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/015b3ee06a804e7da13b5546ed8dcbb3 2024-11-11T06:40:28,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/015b3ee06a804e7da13b5546ed8dcbb3, entries=200, sequenceid=79, filesize=38.6 K 2024-11-11T06:40:28,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/4ec1cf1e010642d48841a0ed9fc6b4cc as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4ec1cf1e010642d48841a0ed9fc6b4cc 2024-11-11T06:40:28,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4ec1cf1e010642d48841a0ed9fc6b4cc, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T06:40:28,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/b6f05480f0554bad9f4fb598ba10a4e2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b6f05480f0554bad9f4fb598ba10a4e2 2024-11-11T06:40:28,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b6f05480f0554bad9f4fb598ba10a4e2, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T06:40:28,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1afd384ae6d17851e72f641429f124f2 in 661ms, sequenceid=79, compaction requested=true 2024-11-11T06:40:28,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:28,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:28,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:28,985 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:28,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:28,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:28,985 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:28,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:28,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:28,987 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:28,987 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/A is initiating minor compaction (all files) 2024-11-11T06:40:28,987 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:28,987 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/B is initiating minor compaction (all files) 2024-11-11T06:40:28,987 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/A in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:28,987 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/B in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:28,987 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/3b5cce1a29884c13aaad4ba900f61e70, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a1bd722f2672475699fb2608b7b040a8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/14d6958cf1ff48af95e4b170d21d8d6c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/015b3ee06a804e7da13b5546ed8dcbb3] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=137.7 K 2024-11-11T06:40:28,987 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5f1d108e6535437ea9f29757d7c103b5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/838f78f627ee4deba9489481a85c5a12, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2fcc6d541b1f4fd1acea26f674145de2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4ec1cf1e010642d48841a0ed9fc6b4cc] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=46.9 K 2024-11-11T06:40:28,987 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:28,988 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/3b5cce1a29884c13aaad4ba900f61e70, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a1bd722f2672475699fb2608b7b040a8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/14d6958cf1ff48af95e4b170d21d8d6c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/015b3ee06a804e7da13b5546ed8dcbb3] 2024-11-11T06:40:28,988 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b5cce1a29884c13aaad4ba900f61e70, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731307225452 2024-11-11T06:40:28,988 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f1d108e6535437ea9f29757d7c103b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731307225452 2024-11-11T06:40:28,989 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 838f78f627ee4deba9489481a85c5a12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731307225506 2024-11-11T06:40:28,989 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1bd722f2672475699fb2608b7b040a8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731307225506 2024-11-11T06:40:28,989 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fcc6d541b1f4fd1acea26f674145de2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307226255 2024-11-11T06:40:28,989 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14d6958cf1ff48af95e4b170d21d8d6c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307226255 2024-11-11T06:40:28,990 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec1cf1e010642d48841a0ed9fc6b4cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731307227675 2024-11-11T06:40:28,990 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 015b3ee06a804e7da13b5546ed8dcbb3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731307227669 2024-11-11T06:40:29,011 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:29,013 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:29,014 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/c410127a6a244a3fbef73376eb6dcf19 is 50, key is test_row_0/B:col10/1731307228322/Put/seqid=0 2024-11-11T06:40:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:29,016 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:29,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:40:29,038 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111b66db8875d694a9c8a676f9743ac733f_1afd384ae6d17851e72f641429f124f2 store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:29,045 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111b66db8875d694a9c8a676f9743ac733f_1afd384ae6d17851e72f641429f124f2, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:29,045 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b66db8875d694a9c8a676f9743ac733f_1afd384ae6d17851e72f641429f124f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:29,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b0c8cfde41f84707b36b027b73a88a45_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307229004/Put/seqid=0 2024-11-11T06:40:29,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741964_1140 (size=12139) 2024-11-11T06:40:29,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741965_1141 (size=4469) 2024-11-11T06:40:29,090 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#A#compaction#120 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:29,091 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/a88539d28a0a4192aaa9d5ced88e979a is 175, key is test_row_0/A:col10/1731307228322/Put/seqid=0 2024-11-11T06:40:29,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741966_1142 (size=12154) 2024-11-11T06:40:29,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741967_1143 (size=31093) 2024-11-11T06:40:29,116 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:29,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b0c8cfde41f84707b36b027b73a88a45_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b0c8cfde41f84707b36b027b73a88a45_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:29,123 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/a88539d28a0a4192aaa9d5ced88e979a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a88539d28a0a4192aaa9d5ced88e979a 2024-11-11T06:40:29,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/59ff620a893d41c39421baa88f7c5336, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:29,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/59ff620a893d41c39421baa88f7c5336 is 175, key is test_row_0/A:col10/1731307229004/Put/seqid=0 2024-11-11T06:40:29,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741968_1144 (size=30955) 2024-11-11T06:40:29,150 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/A of 1afd384ae6d17851e72f641429f124f2 into a88539d28a0a4192aaa9d5ced88e979a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:29,150 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:29,150 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/A, priority=12, startTime=1731307228985; duration=0sec 2024-11-11T06:40:29,150 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:29,150 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:A 2024-11-11T06:40:29,151 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:29,152 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:29,152 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/C is initiating minor compaction (all files) 2024-11-11T06:40:29,152 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/C in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:29,152 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/a4b4f228941e4ba7b36626cff87b9a42, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6b072298643346d3b902e88c5fcc5122, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/de3c2b0ea3be4eacba4df55c28737b42, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b6f05480f0554bad9f4fb598ba10a4e2] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=46.9 K 2024-11-11T06:40:29,153 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b4f228941e4ba7b36626cff87b9a42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731307225452 2024-11-11T06:40:29,154 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b072298643346d3b902e88c5fcc5122, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731307225506 2024-11-11T06:40:29,154 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting de3c2b0ea3be4eacba4df55c28737b42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307226255 2024-11-11T06:40:29,154 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f05480f0554bad9f4fb598ba10a4e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731307227675 2024-11-11T06:40:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307289151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307289151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307289151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,165 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#C#compaction#123 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:29,166 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/be6ea5ca258f4394bf64d196c533bc41 is 50, key is test_row_0/C:col10/1731307228322/Put/seqid=0 2024-11-11T06:40:29,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741969_1145 (size=12139) 2024-11-11T06:40:29,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307289269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307289269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307289269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307289479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307289479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,482 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/c410127a6a244a3fbef73376eb6dcf19 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c410127a6a244a3fbef73376eb6dcf19 2024-11-11T06:40:29,490 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/B of 1afd384ae6d17851e72f641429f124f2 into c410127a6a244a3fbef73376eb6dcf19(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:29,490 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:29,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,490 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/B, priority=12, startTime=1731307228985; duration=0sec 2024-11-11T06:40:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307289489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,490 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:29,490 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:B 2024-11-11T06:40:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/59ff620a893d41c39421baa88f7c5336 2024-11-11T06:40:29,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/749c9dfd1bb2464983e77d2abb143ff3 is 50, key is test_row_0/B:col10/1731307229004/Put/seqid=0 2024-11-11T06:40:29,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741970_1146 (size=12001) 2024-11-11T06:40:29,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/749c9dfd1bb2464983e77d2abb143ff3 2024-11-11T06:40:29,589 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/be6ea5ca258f4394bf64d196c533bc41 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/be6ea5ca258f4394bf64d196c533bc41 2024-11-11T06:40:29,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6616c0c0bf6446d6b0c4ac63688f5553 is 50, key is test_row_0/C:col10/1731307229004/Put/seqid=0 2024-11-11T06:40:29,599 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/C of 1afd384ae6d17851e72f641429f124f2 into be6ea5ca258f4394bf64d196c533bc41(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:29,599 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:29,599 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/C, priority=12, startTime=1731307228985; duration=0sec 2024-11-11T06:40:29,599 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:29,599 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:C 2024-11-11T06:40:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T06:40:29,610 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:29,610 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-11T06:40:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T06:40:29,614 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:29,615 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:29,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:29,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741971_1147 (size=12001) 2024-11-11T06:40:29,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307289699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,701 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1afd384ae6d17851e72f641429f124f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T06:40:29,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T06:40:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:29,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:29,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:29,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307289789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307289790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307289804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307289879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:29,882 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1afd384ae6d17851e72f641429f124f2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T06:40:29,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T06:40:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:29,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:30,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6616c0c0bf6446d6b0c4ac63688f5553 2024-11-11T06:40:30,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/59ff620a893d41c39421baa88f7c5336 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/59ff620a893d41c39421baa88f7c5336 2024-11-11T06:40:30,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/59ff620a893d41c39421baa88f7c5336, entries=150, sequenceid=92, filesize=30.2 K 2024-11-11T06:40:30,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/749c9dfd1bb2464983e77d2abb143ff3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/749c9dfd1bb2464983e77d2abb143ff3 2024-11-11T06:40:30,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/749c9dfd1bb2464983e77d2abb143ff3, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T06:40:30,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6616c0c0bf6446d6b0c4ac63688f5553 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6616c0c0bf6446d6b0c4ac63688f5553 2024-11-11T06:40:30,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6616c0c0bf6446d6b0c4ac63688f5553, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T06:40:30,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 1afd384ae6d17851e72f641429f124f2 in 1048ms, sequenceid=92, compaction requested=false 2024-11-11T06:40:30,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:30,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T06:40:30,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:30,083 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:40:30,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ceed39f477554ceb8a03f8848bc24e84_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307229119/Put/seqid=0 2024-11-11T06:40:30,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741972_1148 (size=12154) 2024-11-11T06:40:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T06:40:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:30,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:30,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307290324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307290326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307290326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307290429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307290439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307290439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:30,514 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ceed39f477554ceb8a03f8848bc24e84_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ceed39f477554ceb8a03f8848bc24e84_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:30,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/68b0217dd62e4d1a9a8101f6a8f71a4b, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:30,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/68b0217dd62e4d1a9a8101f6a8f71a4b is 175, key is test_row_0/A:col10/1731307229119/Put/seqid=0 2024-11-11T06:40:30,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741973_1149 (size=30955) 2024-11-11T06:40:30,525 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/68b0217dd62e4d1a9a8101f6a8f71a4b 2024-11-11T06:40:30,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/51e5a9776f2a4de79f432420c1740911 is 50, key is test_row_0/B:col10/1731307229119/Put/seqid=0 2024-11-11T06:40:30,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741974_1150 (size=12001) 2024-11-11T06:40:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307290639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307290650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307290650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T06:40:30,941 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/51e5a9776f2a4de79f432420c1740911 2024-11-11T06:40:30,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307290950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6851a27fa3914ee189a4a43a002d3130 is 50, key is test_row_0/C:col10/1731307229119/Put/seqid=0 2024-11-11T06:40:30,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307290959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307290959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:30,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741975_1151 (size=12001) 2024-11-11T06:40:30,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6851a27fa3914ee189a4a43a002d3130 2024-11-11T06:40:30,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/68b0217dd62e4d1a9a8101f6a8f71a4b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/68b0217dd62e4d1a9a8101f6a8f71a4b 2024-11-11T06:40:30,997 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/68b0217dd62e4d1a9a8101f6a8f71a4b, entries=150, sequenceid=120, filesize=30.2 K 2024-11-11T06:40:30,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/51e5a9776f2a4de79f432420c1740911 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/51e5a9776f2a4de79f432420c1740911 2024-11-11T06:40:31,014 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/51e5a9776f2a4de79f432420c1740911, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T06:40:31,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/6851a27fa3914ee189a4a43a002d3130 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6851a27fa3914ee189a4a43a002d3130 2024-11-11T06:40:31,021 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6851a27fa3914ee189a4a43a002d3130, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T06:40:31,022 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1afd384ae6d17851e72f641429f124f2 in 939ms, sequenceid=120, compaction requested=true 2024-11-11T06:40:31,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:31,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:31,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-11T06:40:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-11T06:40:31,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T06:40:31,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-11T06:40:31,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4190 sec 2024-11-11T06:40:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:31,463 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:31,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:40:31,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118fd6a6115ca544da89b0a66897ad0856_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307231461/Put/seqid=0 2024-11-11T06:40:31,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741976_1152 (size=14694) 2024-11-11T06:40:31,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307291510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307291513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307291513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307291619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307291629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307291629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T06:40:31,760 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:31,760 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T06:40:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T06:40:31,762 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:31,762 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:31,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:31,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307291829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307291839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307291839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T06:40:31,877 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:31,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118fd6a6115ca544da89b0a66897ad0856_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118fd6a6115ca544da89b0a66897ad0856_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:31,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/607e1ffd19924115bda4da52f7fa698f, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:31,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/607e1ffd19924115bda4da52f7fa698f is 175, key is test_row_0/A:col10/1731307231461/Put/seqid=0 2024-11-11T06:40:31,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741977_1153 (size=39649) 2024-11-11T06:40:31,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307291899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:31,905 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1afd384ae6d17851e72f641429f124f2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:31,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T06:40:31,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:31,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:31,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:31,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:31,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T06:40:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T06:40:32,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307292138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307292149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307292149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T06:40:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,289 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/607e1ffd19924115bda4da52f7fa698f 2024-11-11T06:40:32,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/37208ef2832249cbaadac0176a045a61 is 50, key is test_row_0/B:col10/1731307231461/Put/seqid=0 2024-11-11T06:40:32,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741978_1154 (size=12051) 2024-11-11T06:40:32,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/37208ef2832249cbaadac0176a045a61 2024-11-11T06:40:32,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/2651d1dae1a04626b7cc832843e310aa is 50, key is test_row_0/C:col10/1731307231461/Put/seqid=0 2024-11-11T06:40:32,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741979_1155 (size=12051) 2024-11-11T06:40:32,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/2651d1dae1a04626b7cc832843e310aa 2024-11-11T06:40:32,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/607e1ffd19924115bda4da52f7fa698f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/607e1ffd19924115bda4da52f7fa698f 2024-11-11T06:40:32,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/607e1ffd19924115bda4da52f7fa698f, entries=200, sequenceid=132, filesize=38.7 K 2024-11-11T06:40:32,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/37208ef2832249cbaadac0176a045a61 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/37208ef2832249cbaadac0176a045a61 2024-11-11T06:40:32,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/37208ef2832249cbaadac0176a045a61, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T06:40:32,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/2651d1dae1a04626b7cc832843e310aa as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2651d1dae1a04626b7cc832843e310aa 2024-11-11T06:40:32,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T06:40:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2651d1dae1a04626b7cc832843e310aa, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T06:40:32,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1afd384ae6d17851e72f641429f124f2 in 911ms, sequenceid=132, compaction requested=true 2024-11-11T06:40:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:32,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:32,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:32,375 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:32,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:32,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:32,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:32,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:32,376 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:32,377 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:32,377 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132652 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:32,377 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/A is initiating minor compaction (all files) 2024-11-11T06:40:32,377 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/B is initiating minor compaction (all files) 2024-11-11T06:40:32,377 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/B in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,377 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/A in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,378 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c410127a6a244a3fbef73376eb6dcf19, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/749c9dfd1bb2464983e77d2abb143ff3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/51e5a9776f2a4de79f432420c1740911, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/37208ef2832249cbaadac0176a045a61] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=47.1 K 2024-11-11T06:40:32,378 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a88539d28a0a4192aaa9d5ced88e979a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/59ff620a893d41c39421baa88f7c5336, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/68b0217dd62e4d1a9a8101f6a8f71a4b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/607e1ffd19924115bda4da52f7fa698f] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=129.5 K 2024-11-11T06:40:32,378 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,378 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a88539d28a0a4192aaa9d5ced88e979a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/59ff620a893d41c39421baa88f7c5336, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/68b0217dd62e4d1a9a8101f6a8f71a4b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/607e1ffd19924115bda4da52f7fa698f] 2024-11-11T06:40:32,378 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c410127a6a244a3fbef73376eb6dcf19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731307227675 2024-11-11T06:40:32,378 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a88539d28a0a4192aaa9d5ced88e979a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731307227675 2024-11-11T06:40:32,378 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 749c9dfd1bb2464983e77d2abb143ff3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307228996 2024-11-11T06:40:32,378 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ff620a893d41c39421baa88f7c5336, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307228996 2024-11-11T06:40:32,379 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e5a9776f2a4de79f432420c1740911, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731307229102 2024-11-11T06:40:32,379 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b0217dd62e4d1a9a8101f6a8f71a4b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731307229102 2024-11-11T06:40:32,380 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 607e1ffd19924115bda4da52f7fa698f, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307230305 2024-11-11T06:40:32,380 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 37208ef2832249cbaadac0176a045a61, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307230305 2024-11-11T06:40:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T06:40:32,389 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:32,392 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111613ab0d237e2408f885859d01de6abbf_1afd384ae6d17851e72f641429f124f2 store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:32,393 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:32,393 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/c1535e9912e147acb1147af2d85e4455 is 50, key is test_row_0/B:col10/1731307231461/Put/seqid=0 2024-11-11T06:40:32,398 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111613ab0d237e2408f885859d01de6abbf_1afd384ae6d17851e72f641429f124f2, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:32,398 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111613ab0d237e2408f885859d01de6abbf_1afd384ae6d17851e72f641429f124f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:32,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741980_1156 (size=12325) 2024-11-11T06:40:32,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741981_1157 (size=4469) 2024-11-11T06:40:32,410 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#A#compaction#132 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:32,411 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/06f3d5d6210c499c99b9e1a83303e753 is 175, key is test_row_0/A:col10/1731307231461/Put/seqid=0 2024-11-11T06:40:32,413 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/c1535e9912e147acb1147af2d85e4455 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c1535e9912e147acb1147af2d85e4455 2024-11-11T06:40:32,418 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/B of 1afd384ae6d17851e72f641429f124f2 into c1535e9912e147acb1147af2d85e4455(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:32,419 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:32,419 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/B, priority=12, startTime=1731307232375; duration=0sec 2024-11-11T06:40:32,419 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:32,419 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:B 2024-11-11T06:40:32,419 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:32,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741982_1158 (size=31279) 2024-11-11T06:40:32,421 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:32,421 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/C is initiating minor compaction (all files) 2024-11-11T06:40:32,421 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/C in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,421 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/be6ea5ca258f4394bf64d196c533bc41, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6616c0c0bf6446d6b0c4ac63688f5553, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6851a27fa3914ee189a4a43a002d3130, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2651d1dae1a04626b7cc832843e310aa] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=47.1 K 2024-11-11T06:40:32,421 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting be6ea5ca258f4394bf64d196c533bc41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731307227675 2024-11-11T06:40:32,422 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 6616c0c0bf6446d6b0c4ac63688f5553, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307228996 2024-11-11T06:40:32,422 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 6851a27fa3914ee189a4a43a002d3130, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731307229102 2024-11-11T06:40:32,422 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2651d1dae1a04626b7cc832843e310aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307230305 2024-11-11T06:40:32,446 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#C#compaction#134 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:32,446 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/0f4775c9f09446fca561e1f18a7b8840 is 50, key is test_row_0/C:col10/1731307231461/Put/seqid=0 2024-11-11T06:40:32,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741983_1159 (size=12325) 2024-11-11T06:40:32,455 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/0f4775c9f09446fca561e1f18a7b8840 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/0f4775c9f09446fca561e1f18a7b8840 2024-11-11T06:40:32,461 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/C of 1afd384ae6d17851e72f641429f124f2 into 0f4775c9f09446fca561e1f18a7b8840(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:32,461 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:32,461 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/C, priority=12, startTime=1731307232376; duration=0sec 2024-11-11T06:40:32,461 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:32,461 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:C 2024-11-11T06:40:32,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T06:40:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:32,527 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:40:32,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f4e5aebfcc1c488f88bc683e2c0755bc_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307231508/Put/seqid=0 2024-11-11T06:40:32,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741984_1160 (size=12304) 2024-11-11T06:40:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:32,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:32,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307292693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307292696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307292696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307292799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307292809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307292810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:32,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:32,829 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/06f3d5d6210c499c99b9e1a83303e753 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/06f3d5d6210c499c99b9e1a83303e753 2024-11-11T06:40:32,836 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/A of 1afd384ae6d17851e72f641429f124f2 into 06f3d5d6210c499c99b9e1a83303e753(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:32,836 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:32,836 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/A, priority=12, startTime=1731307232374; duration=0sec 2024-11-11T06:40:32,836 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:32,836 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:A 2024-11-11T06:40:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T06:40:32,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:32,944 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f4e5aebfcc1c488f88bc683e2c0755bc_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4e5aebfcc1c488f88bc683e2c0755bc_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:32,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/297f14fd8f16448baad0d5e2f9f1abfd, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:32,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/297f14fd8f16448baad0d5e2f9f1abfd is 175, key is test_row_0/A:col10/1731307231508/Put/seqid=0 2024-11-11T06:40:32,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741985_1161 (size=31105) 2024-11-11T06:40:32,953 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/297f14fd8f16448baad0d5e2f9f1abfd 2024-11-11T06:40:32,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/2d3adae3281b4ccfa99bee4ae968705d is 50, key is test_row_0/B:col10/1731307231508/Put/seqid=0 2024-11-11T06:40:32,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741986_1162 (size=12151) 2024-11-11T06:40:33,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307293009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307293020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307293020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T06:40:33,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T06:40:33,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307293319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307293329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307293329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,397 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/2d3adae3281b4ccfa99bee4ae968705d 2024-11-11T06:40:33,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/d821e76e05434b21a6f928ab557fa098 is 50, key is test_row_0/C:col10/1731307231508/Put/seqid=0 2024-11-11T06:40:33,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741987_1163 (size=12151) 2024-11-11T06:40:33,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307293729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,732 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1afd384ae6d17851e72f641429f124f2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,823 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/d821e76e05434b21a6f928ab557fa098 2024-11-11T06:40:33,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307293830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307293839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307293839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:33,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/297f14fd8f16448baad0d5e2f9f1abfd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/297f14fd8f16448baad0d5e2f9f1abfd 2024-11-11T06:40:33,849 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/297f14fd8f16448baad0d5e2f9f1abfd, entries=150, sequenceid=159, filesize=30.4 K 2024-11-11T06:40:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/2d3adae3281b4ccfa99bee4ae968705d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2d3adae3281b4ccfa99bee4ae968705d 2024-11-11T06:40:33,855 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2d3adae3281b4ccfa99bee4ae968705d, entries=150, sequenceid=159, filesize=11.9 K 2024-11-11T06:40:33,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/d821e76e05434b21a6f928ab557fa098 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/d821e76e05434b21a6f928ab557fa098 2024-11-11T06:40:33,863 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/d821e76e05434b21a6f928ab557fa098, entries=150, sequenceid=159, filesize=11.9 K 2024-11-11T06:40:33,864 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1afd384ae6d17851e72f641429f124f2 in 1337ms, sequenceid=159, compaction requested=false 2024-11-11T06:40:33,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:33,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:33,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T06:40:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T06:40:33,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T06:40:33,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1040 sec 2024-11-11T06:40:33,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.1120 sec 2024-11-11T06:40:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T06:40:33,909 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:33,910 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T06:40:33,912 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T06:40:33,915 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:33,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T06:40:34,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T06:40:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:34,069 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:40:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f89238dc8fc4640992ce3642d17f268_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307232695/Put/seqid=0 2024-11-11T06:40:34,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741988_1164 (size=12304) 2024-11-11T06:40:34,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,128 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f89238dc8fc4640992ce3642d17f268_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f89238dc8fc4640992ce3642d17f268_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:34,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/61900b6df4fc405f926a404006dc1838, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:34,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/61900b6df4fc405f926a404006dc1838 is 175, key is test_row_0/A:col10/1731307232695/Put/seqid=0 2024-11-11T06:40:34,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741989_1165 (size=31105) 2024-11-11T06:40:34,164 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/61900b6df4fc405f926a404006dc1838 2024-11-11T06:40:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/c5092c787c93439f95adc60510bb8a20 is 50, key is test_row_0/B:col10/1731307232695/Put/seqid=0 2024-11-11T06:40:34,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741990_1166 (size=12151) 2024-11-11T06:40:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T06:40:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T06:40:34,595 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/c5092c787c93439f95adc60510bb8a20 2024-11-11T06:40:34,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/9769972335db43e4b1000fcd6cd2e9dd is 50, key is test_row_0/C:col10/1731307232695/Put/seqid=0 2024-11-11T06:40:34,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741991_1167 (size=12151) 2024-11-11T06:40:34,624 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/9769972335db43e4b1000fcd6cd2e9dd 2024-11-11T06:40:34,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/61900b6df4fc405f926a404006dc1838 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/61900b6df4fc405f926a404006dc1838 2024-11-11T06:40:34,640 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/61900b6df4fc405f926a404006dc1838, entries=150, sequenceid=172, filesize=30.4 K 2024-11-11T06:40:34,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/c5092c787c93439f95adc60510bb8a20 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c5092c787c93439f95adc60510bb8a20 2024-11-11T06:40:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,648 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c5092c787c93439f95adc60510bb8a20, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T06:40:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/9769972335db43e4b1000fcd6cd2e9dd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/9769972335db43e4b1000fcd6cd2e9dd 2024-11-11T06:40:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,657 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/9769972335db43e4b1000fcd6cd2e9dd, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T06:40:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,658 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 1afd384ae6d17851e72f641429f124f2 in 589ms, sequenceid=172, compaction requested=true 2024-11-11T06:40:34,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:34,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:34,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T06:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T06:40:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T06:40:34,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 745 msec 2024-11-11T06:40:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 754 msec 2024-11-11T06:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:35,001 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:35,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:40:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f7052a612ab469191c46e082ef5688d_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307234992/Put/seqid=0 2024-11-11T06:40:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T06:40:35,049 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,049 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T06:40:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,051 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T06:40:35,055 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741993_1169 (size=24758) 2024-11-11T06:40:35,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307295081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307295082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307295084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T06:40:35,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307295189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307295189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307295199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T06:40:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T06:40:35,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T06:40:35,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307295399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307295399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307295409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,468 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:35,474 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f7052a612ab469191c46e082ef5688d_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f7052a612ab469191c46e082ef5688d_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:35,475 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f70515fdd6874bd0bc58d5031413a87e, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:35,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f70515fdd6874bd0bc58d5031413a87e is 175, key is test_row_0/A:col10/1731307234992/Put/seqid=0 2024-11-11T06:40:35,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741992_1168 (size=74394) 2024-11-11T06:40:35,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f70515fdd6874bd0bc58d5031413a87e 2024-11-11T06:40:35,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/5cc3b4304c05463790231104ec66c701 is 50, key is test_row_0/B:col10/1731307234992/Put/seqid=0 2024-11-11T06:40:35,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741994_1170 (size=12151) 2024-11-11T06:40:35,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/5cc3b4304c05463790231104ec66c701 2024-11-11T06:40:35,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/18f15c0680514d019b6f3c664f562de3 is 50, key is test_row_0/C:col10/1731307234992/Put/seqid=0 2024-11-11T06:40:35,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T06:40:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:35,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741995_1171 (size=12151) 2024-11-11T06:40:35,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/18f15c0680514d019b6f3c664f562de3 2024-11-11T06:40:35,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f70515fdd6874bd0bc58d5031413a87e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f70515fdd6874bd0bc58d5031413a87e 2024-11-11T06:40:35,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f70515fdd6874bd0bc58d5031413a87e, entries=400, sequenceid=183, filesize=72.7 K 2024-11-11T06:40:35,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/5cc3b4304c05463790231104ec66c701 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5cc3b4304c05463790231104ec66c701 2024-11-11T06:40:35,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5cc3b4304c05463790231104ec66c701, entries=150, sequenceid=183, filesize=11.9 K 2024-11-11T06:40:35,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/18f15c0680514d019b6f3c664f562de3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/18f15c0680514d019b6f3c664f562de3 2024-11-11T06:40:35,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/18f15c0680514d019b6f3c664f562de3, entries=150, sequenceid=183, filesize=11.9 K 2024-11-11T06:40:35,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1afd384ae6d17851e72f641429f124f2 in 570ms, sequenceid=183, compaction requested=true 2024-11-11T06:40:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:35,572 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T06:40:35,572 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:35,575 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:35,575 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/B is initiating minor compaction (all files) 2024-11-11T06:40:35,575 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/B in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,576 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c1535e9912e147acb1147af2d85e4455, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2d3adae3281b4ccfa99bee4ae968705d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c5092c787c93439f95adc60510bb8a20, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5cc3b4304c05463790231104ec66c701] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=47.6 K 2024-11-11T06:40:35,576 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167883 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:35,576 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/A is initiating minor compaction (all files) 2024-11-11T06:40:35,576 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/A in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,576 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/06f3d5d6210c499c99b9e1a83303e753, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/297f14fd8f16448baad0d5e2f9f1abfd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/61900b6df4fc405f926a404006dc1838, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f70515fdd6874bd0bc58d5031413a87e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=163.9 K 2024-11-11T06:40:35,576 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,576 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/06f3d5d6210c499c99b9e1a83303e753, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/297f14fd8f16448baad0d5e2f9f1abfd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/61900b6df4fc405f926a404006dc1838, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f70515fdd6874bd0bc58d5031413a87e] 2024-11-11T06:40:35,577 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f3d5d6210c499c99b9e1a83303e753, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307230305 2024-11-11T06:40:35,577 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c1535e9912e147acb1147af2d85e4455, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307230305 2024-11-11T06:40:35,577 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 297f14fd8f16448baad0d5e2f9f1abfd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731307231506 2024-11-11T06:40:35,577 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61900b6df4fc405f926a404006dc1838, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307232692 2024-11-11T06:40:35,577 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3adae3281b4ccfa99bee4ae968705d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731307231506 2024-11-11T06:40:35,578 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting f70515fdd6874bd0bc58d5031413a87e, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307234912 2024-11-11T06:40:35,578 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c5092c787c93439f95adc60510bb8a20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307232692 2024-11-11T06:40:35,578 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cc3b4304c05463790231104ec66c701, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307234992 2024-11-11T06:40:35,598 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:35,598 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/936767186b7a4f349c8493eef7c0f834 is 50, key is test_row_0/B:col10/1731307234992/Put/seqid=0 2024-11-11T06:40:35,599 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:35,611 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411116567d43a10a14cef841eaaeeeffab55e_1afd384ae6d17851e72f641429f124f2 store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:35,615 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411116567d43a10a14cef841eaaeeeffab55e_1afd384ae6d17851e72f641429f124f2, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:35,615 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116567d43a10a14cef841eaaeeeffab55e_1afd384ae6d17851e72f641429f124f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:35,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741996_1172 (size=12561) 2024-11-11T06:40:35,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T06:40:35,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:35,676 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:40:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T06:40:35,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741997_1173 (size=4469) 2024-11-11T06:40:35,693 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#A#compaction#145 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:35,694 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/ccd2dc52dcd04f4da26609a6d06c33ec is 175, key is test_row_0/A:col10/1731307234992/Put/seqid=0 2024-11-11T06:40:35,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111134af525fc6ac43cd9ff2e932ea52116c_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307235076/Put/seqid=0 2024-11-11T06:40:35,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:35,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741998_1174 (size=31515) 2024-11-11T06:40:35,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307295737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307295741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307295754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741999_1175 (size=12304) 2024-11-11T06:40:35,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307295859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307295859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307295869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:35,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307295929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:35,935 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1afd384ae6d17851e72f641429f124f2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307296069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307296069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,074 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/936767186b7a4f349c8493eef7c0f834 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/936767186b7a4f349c8493eef7c0f834 2024-11-11T06:40:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307296079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,080 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/B of 1afd384ae6d17851e72f641429f124f2 into 936767186b7a4f349c8493eef7c0f834(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:36,080 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:36,080 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/B, priority=12, startTime=1731307235572; duration=0sec 2024-11-11T06:40:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,081 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:36,081 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:B 2024-11-11T06:40:36,081 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:40:36,083 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:40:36,083 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/C is initiating minor compaction (all files) 2024-11-11T06:40:36,083 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/C in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:36,083 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/0f4775c9f09446fca561e1f18a7b8840, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/d821e76e05434b21a6f928ab557fa098, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/9769972335db43e4b1000fcd6cd2e9dd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/18f15c0680514d019b6f3c664f562de3] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=47.6 K 2024-11-11T06:40:36,084 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4775c9f09446fca561e1f18a7b8840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307230305 2024-11-11T06:40:36,084 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting d821e76e05434b21a6f928ab557fa098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731307231506 2024-11-11T06:40:36,084 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 9769972335db43e4b1000fcd6cd2e9dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307232692 2024-11-11T06:40:36,085 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f15c0680514d019b6f3c664f562de3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307234992 2024-11-11T06:40:36,111 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:36,111 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/620041b039224d33aa25c258ae7028b4 is 50, key is test_row_0/C:col10/1731307234992/Put/seqid=0 2024-11-11T06:40:36,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742000_1176 (size=12561) 2024-11-11T06:40:36,141 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/620041b039224d33aa25c258ae7028b4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/620041b039224d33aa25c258ae7028b4 2024-11-11T06:40:36,152 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/C of 1afd384ae6d17851e72f641429f124f2 into 620041b039224d33aa25c258ae7028b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:36,152 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:36,153 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/C, priority=12, startTime=1731307235572; duration=0sec 2024-11-11T06:40:36,153 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:36,153 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:C 2024-11-11T06:40:36,154 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/ccd2dc52dcd04f4da26609a6d06c33ec as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/ccd2dc52dcd04f4da26609a6d06c33ec 2024-11-11T06:40:36,166 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/A of 1afd384ae6d17851e72f641429f124f2 into ccd2dc52dcd04f4da26609a6d06c33ec(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:36,166 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:36,166 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/A, priority=12, startTime=1731307235572; duration=0sec 2024-11-11T06:40:36,166 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:36,166 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:A 2024-11-11T06:40:36,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:36,179 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111134af525fc6ac43cd9ff2e932ea52116c_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111134af525fc6ac43cd9ff2e932ea52116c_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/0e0520ace97945f1b59ebf206831572f, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:36,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/0e0520ace97945f1b59ebf206831572f is 175, key is test_row_0/A:col10/1731307235076/Put/seqid=0 2024-11-11T06:40:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T06:40:36,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742001_1177 (size=31105) 2024-11-11T06:40:36,214 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/0e0520ace97945f1b59ebf206831572f 2024-11-11T06:40:36,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/78998379c5374c4fa2eaf1cea947458e is 50, key is test_row_0/B:col10/1731307235076/Put/seqid=0 2024-11-11T06:40:36,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742002_1178 (size=12151) 2024-11-11T06:40:36,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307296379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307296379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307296389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,665 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/78998379c5374c4fa2eaf1cea947458e 2024-11-11T06:40:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/8a7d1c84811b43d297d0035d96a311a1 is 50, key is test_row_0/C:col10/1731307235076/Put/seqid=0 2024-11-11T06:40:36,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742003_1179 (size=12151) 2024-11-11T06:40:36,697 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/8a7d1c84811b43d297d0035d96a311a1 2024-11-11T06:40:36,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/0e0520ace97945f1b59ebf206831572f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/0e0520ace97945f1b59ebf206831572f 2024-11-11T06:40:36,712 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/0e0520ace97945f1b59ebf206831572f, entries=150, sequenceid=208, filesize=30.4 K 2024-11-11T06:40:36,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/78998379c5374c4fa2eaf1cea947458e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/78998379c5374c4fa2eaf1cea947458e 2024-11-11T06:40:36,722 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/78998379c5374c4fa2eaf1cea947458e, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T06:40:36,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/8a7d1c84811b43d297d0035d96a311a1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/8a7d1c84811b43d297d0035d96a311a1 2024-11-11T06:40:36,732 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/8a7d1c84811b43d297d0035d96a311a1, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T06:40:36,733 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 1afd384ae6d17851e72f641429f124f2 in 1058ms, sequenceid=208, compaction requested=false 2024-11-11T06:40:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T06:40:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T06:40:36,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T06:40:36,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-11T06:40:36,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.6990 sec 2024-11-11T06:40:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:36,893 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:36,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:40:36,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111193494a067d7d41d4b1c7d81ae9931084_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307235738/Put/seqid=0 2024-11-11T06:40:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307296951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307296952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307296953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:36,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:36,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742004_1180 (size=17284) 2024-11-11T06:40:36,974 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:36,979 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111193494a067d7d41d4b1c7d81ae9931084_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193494a067d7d41d4b1c7d81ae9931084_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:36,980 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/2760f05cee9c4f26af4640aa22910d57, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:36,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/2760f05cee9c4f26af4640aa22910d57 is 175, key is test_row_0/A:col10/1731307235738/Put/seqid=0 2024-11-11T06:40:36,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742005_1181 (size=48389) 2024-11-11T06:40:37,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307297069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307297069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307297069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T06:40:37,199 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:37,200 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T06:40:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T06:40:37,201 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:37,202 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:37,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:37,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307297279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307297279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307297279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T06:40:37,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T06:40:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,389 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/2760f05cee9c4f26af4640aa22910d57 2024-11-11T06:40:37,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/119da708bd9042a8bdbfe797e81afb1a is 50, key is test_row_0/B:col10/1731307235738/Put/seqid=0 2024-11-11T06:40:37,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742006_1182 (size=12151) 2024-11-11T06:40:37,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T06:40:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T06:40:37,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307297589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307297589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307297589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:37,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T06:40:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/119da708bd9042a8bdbfe797e81afb1a 2024-11-11T06:40:37,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/4577e9056cb744f0bff512a44dbc6848 is 50, key is test_row_0/C:col10/1731307235738/Put/seqid=0 2024-11-11T06:40:37,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T06:40:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T06:40:37,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742007_1183 (size=12151) 2024-11-11T06:40:37,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/4577e9056cb744f0bff512a44dbc6848 2024-11-11T06:40:37,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/2760f05cee9c4f26af4640aa22910d57 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/2760f05cee9c4f26af4640aa22910d57 2024-11-11T06:40:37,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/2760f05cee9c4f26af4640aa22910d57, entries=250, sequenceid=225, filesize=47.3 K 2024-11-11T06:40:37,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/119da708bd9042a8bdbfe797e81afb1a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/119da708bd9042a8bdbfe797e81afb1a 2024-11-11T06:40:37,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/119da708bd9042a8bdbfe797e81afb1a, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T06:40:37,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/4577e9056cb744f0bff512a44dbc6848 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/4577e9056cb744f0bff512a44dbc6848 2024-11-11T06:40:37,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/4577e9056cb744f0bff512a44dbc6848, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T06:40:37,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 1afd384ae6d17851e72f641429f124f2 in 971ms, sequenceid=225, compaction requested=true 2024-11-11T06:40:37,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:37,864 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:37,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:37,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:37,864 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:37,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:37,867 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:37,867 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/B is initiating minor compaction (all files) 2024-11-11T06:40:37,867 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:37,867 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/B in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,867 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/A is initiating minor compaction (all files) 2024-11-11T06:40:37,867 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/A in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,867 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/936767186b7a4f349c8493eef7c0f834, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/78998379c5374c4fa2eaf1cea947458e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/119da708bd9042a8bdbfe797e81afb1a] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=36.0 K 2024-11-11T06:40:37,867 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/ccd2dc52dcd04f4da26609a6d06c33ec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/0e0520ace97945f1b59ebf206831572f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/2760f05cee9c4f26af4640aa22910d57] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=108.4 K 2024-11-11T06:40:37,867 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,867 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/ccd2dc52dcd04f4da26609a6d06c33ec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/0e0520ace97945f1b59ebf206831572f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/2760f05cee9c4f26af4640aa22910d57] 2024-11-11T06:40:37,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:37,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:37,867 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 936767186b7a4f349c8493eef7c0f834, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307234992 2024-11-11T06:40:37,868 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccd2dc52dcd04f4da26609a6d06c33ec, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307234992 2024-11-11T06:40:37,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:37,868 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 78998379c5374c4fa2eaf1cea947458e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307235076 2024-11-11T06:40:37,868 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e0520ace97945f1b59ebf206831572f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307235076 2024-11-11T06:40:37,869 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2760f05cee9c4f26af4640aa22910d57, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731307235725 2024-11-11T06:40:37,869 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 119da708bd9042a8bdbfe797e81afb1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731307235728 2024-11-11T06:40:37,884 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:37,887 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/11225b66deac480aacaf8fec8e1b8e93 is 50, key is test_row_0/B:col10/1731307235738/Put/seqid=0 2024-11-11T06:40:37,889 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:37,904 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111fd11b4ca8d01460bb5f70cd4fe59ab6e_1afd384ae6d17851e72f641429f124f2 store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:37,906 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111fd11b4ca8d01460bb5f70cd4fe59ab6e_1afd384ae6d17851e72f641429f124f2, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:37,906 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fd11b4ca8d01460bb5f70cd4fe59ab6e_1afd384ae6d17851e72f641429f124f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:37,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742008_1184 (size=12663) 2024-11-11T06:40:37,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742009_1185 (size=4469) 2024-11-11T06:40:37,957 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#A#compaction#154 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:37,958 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/81d01942301946b3b4e1fb20bea8dd6b is 175, key is test_row_0/A:col10/1731307235738/Put/seqid=0 2024-11-11T06:40:37,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T06:40:37,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,969 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T06:40:37,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742010_1186 (size=31617) 2024-11-11T06:40:37,979 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/81d01942301946b3b4e1fb20bea8dd6b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/81d01942301946b3b4e1fb20bea8dd6b 2024-11-11T06:40:37,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b11d11acb06b4b90a88b6fa4c10d3ee1_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307236949/Put/seqid=0 2024-11-11T06:40:37,991 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/A of 1afd384ae6d17851e72f641429f124f2 into 81d01942301946b3b4e1fb20bea8dd6b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:37,991 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:37,991 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/A, priority=13, startTime=1731307237864; duration=0sec 2024-11-11T06:40:37,992 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:37,992 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:A 2024-11-11T06:40:37,992 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:37,994 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:37,994 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/C is initiating minor compaction (all files) 2024-11-11T06:40:37,994 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/C in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:37,994 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/620041b039224d33aa25c258ae7028b4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/8a7d1c84811b43d297d0035d96a311a1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/4577e9056cb744f0bff512a44dbc6848] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=36.0 K 2024-11-11T06:40:37,995 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 620041b039224d33aa25c258ae7028b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307234992 2024-11-11T06:40:37,996 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7d1c84811b43d297d0035d96a311a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307235076 2024-11-11T06:40:37,996 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4577e9056cb744f0bff512a44dbc6848, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731307235728 2024-11-11T06:40:38,028 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:38,028 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/ba819cebf008416c822f5ea51f09e264 is 50, key is test_row_0/C:col10/1731307235738/Put/seqid=0 2024-11-11T06:40:38,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742011_1187 (size=12304) 2024-11-11T06:40:38,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742012_1188 (size=12663) 2024-11-11T06:40:38,060 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/ba819cebf008416c822f5ea51f09e264 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ba819cebf008416c822f5ea51f09e264 2024-11-11T06:40:38,067 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/C of 1afd384ae6d17851e72f641429f124f2 into ba819cebf008416c822f5ea51f09e264(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:38,067 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:38,067 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/C, priority=13, startTime=1731307237867; duration=0sec 2024-11-11T06:40:38,068 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:38,068 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:C 2024-11-11T06:40:38,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:38,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307298116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307298120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307298120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307298229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307298229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307298229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T06:40:38,351 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/11225b66deac480aacaf8fec8e1b8e93 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/11225b66deac480aacaf8fec8e1b8e93 2024-11-11T06:40:38,357 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/B of 1afd384ae6d17851e72f641429f124f2 into 11225b66deac480aacaf8fec8e1b8e93(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:38,357 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:38,357 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/B, priority=13, startTime=1731307237864; duration=0sec 2024-11-11T06:40:38,358 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:38,358 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:B 2024-11-11T06:40:38,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307298440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307298440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307298440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:38,457 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b11d11acb06b4b90a88b6fa4c10d3ee1_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b11d11acb06b4b90a88b6fa4c10d3ee1_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/52a1c13f9bf64c7ab0a451b303a6f49a, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:38,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/52a1c13f9bf64c7ab0a451b303a6f49a is 175, key is test_row_0/A:col10/1731307236949/Put/seqid=0 2024-11-11T06:40:38,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742013_1189 (size=31105) 2024-11-11T06:40:38,464 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/52a1c13f9bf64c7ab0a451b303a6f49a 2024-11-11T06:40:38,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/1bb626be39004f76a4f3bedeb9e6898e is 50, key is test_row_0/B:col10/1731307236949/Put/seqid=0 2024-11-11T06:40:38,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742014_1190 (size=12151) 2024-11-11T06:40:38,532 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/1bb626be39004f76a4f3bedeb9e6898e 2024-11-11T06:40:38,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/ee50d004851442d1b7a92b1afc35f2bd is 50, key is test_row_0/C:col10/1731307236949/Put/seqid=0 2024-11-11T06:40:38,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742015_1191 (size=12151) 2024-11-11T06:40:38,594 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/ee50d004851442d1b7a92b1afc35f2bd 2024-11-11T06:40:38,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/52a1c13f9bf64c7ab0a451b303a6f49a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/52a1c13f9bf64c7ab0a451b303a6f49a 2024-11-11T06:40:38,609 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/52a1c13f9bf64c7ab0a451b303a6f49a, entries=150, sequenceid=248, filesize=30.4 K 2024-11-11T06:40:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/1bb626be39004f76a4f3bedeb9e6898e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/1bb626be39004f76a4f3bedeb9e6898e 2024-11-11T06:40:38,635 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/1bb626be39004f76a4f3bedeb9e6898e, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T06:40:38,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/ee50d004851442d1b7a92b1afc35f2bd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ee50d004851442d1b7a92b1afc35f2bd 2024-11-11T06:40:38,644 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ee50d004851442d1b7a92b1afc35f2bd, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T06:40:38,645 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1afd384ae6d17851e72f641429f124f2 in 676ms, sequenceid=248, compaction requested=false 2024-11-11T06:40:38,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:38,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:38,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T06:40:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T06:40:38,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T06:40:38,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-11T06:40:38,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4520 sec 2024-11-11T06:40:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:38,761 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:38,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T06:40:38,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117711ce0f11c248c88b7b52e30569c0ff_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307238118/Put/seqid=0 2024-11-11T06:40:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307298815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742016_1192 (size=12454) 2024-11-11T06:40:38,823 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:38,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307298816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307298816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,829 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117711ce0f11c248c88b7b52e30569c0ff_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117711ce0f11c248c88b7b52e30569c0ff_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:38,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/4d0fd2e8e53d444d90c0065e46533582, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:38,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/4d0fd2e8e53d444d90c0065e46533582 is 175, key is test_row_0/A:col10/1731307238118/Put/seqid=0 2024-11-11T06:40:38,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742017_1193 (size=31255) 2024-11-11T06:40:38,847 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/4d0fd2e8e53d444d90c0065e46533582 2024-11-11T06:40:38,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/b2d37eeb38044c3db23e7419e15977fb is 50, key is test_row_0/B:col10/1731307238118/Put/seqid=0 2024-11-11T06:40:38,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742018_1194 (size=12301) 2024-11-11T06:40:38,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307298919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307298929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307298929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307299129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307299139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307299139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/b2d37eeb38044c3db23e7419e15977fb 2024-11-11T06:40:39,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/7afa3bc0ba974970b9353d3fde0de173 is 50, key is test_row_0/C:col10/1731307238118/Put/seqid=0 2024-11-11T06:40:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T06:40:39,349 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:39,350 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T06:40:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T06:40:39,353 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:39,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742019_1195 (size=12301) 2024-11-11T06:40:39,354 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:39,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:39,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307299439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307299449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307299449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T06:40:39,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T06:40:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:39,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T06:40:39,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:39,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:39,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T06:40:39,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/7afa3bc0ba974970b9353d3fde0de173 2024-11-11T06:40:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/4d0fd2e8e53d444d90c0065e46533582 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/4d0fd2e8e53d444d90c0065e46533582 2024-11-11T06:40:39,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/4d0fd2e8e53d444d90c0065e46533582, entries=150, sequenceid=267, filesize=30.5 K 2024-11-11T06:40:39,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/b2d37eeb38044c3db23e7419e15977fb as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/b2d37eeb38044c3db23e7419e15977fb 2024-11-11T06:40:39,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/b2d37eeb38044c3db23e7419e15977fb, entries=150, sequenceid=267, filesize=12.0 K 2024-11-11T06:40:39,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/7afa3bc0ba974970b9353d3fde0de173 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/7afa3bc0ba974970b9353d3fde0de173 2024-11-11T06:40:39,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/7afa3bc0ba974970b9353d3fde0de173, entries=150, sequenceid=267, filesize=12.0 K 2024-11-11T06:40:39,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 1afd384ae6d17851e72f641429f124f2 in 1036ms, sequenceid=267, compaction requested=true 2024-11-11T06:40:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:39,798 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:39,798 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:39,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:39,799 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:39,799 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93977 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:39,799 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/B is initiating minor compaction (all files) 2024-11-11T06:40:39,799 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/A is initiating minor compaction (all files) 2024-11-11T06:40:39,800 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/B in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,800 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/A in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,800 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/81d01942301946b3b4e1fb20bea8dd6b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/52a1c13f9bf64c7ab0a451b303a6f49a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/4d0fd2e8e53d444d90c0065e46533582] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=91.8 K 2024-11-11T06:40:39,800 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/11225b66deac480aacaf8fec8e1b8e93, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/1bb626be39004f76a4f3bedeb9e6898e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/b2d37eeb38044c3db23e7419e15977fb] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=36.2 K 2024-11-11T06:40:39,800 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,800 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/81d01942301946b3b4e1fb20bea8dd6b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/52a1c13f9bf64c7ab0a451b303a6f49a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/4d0fd2e8e53d444d90c0065e46533582] 2024-11-11T06:40:39,800 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 11225b66deac480aacaf8fec8e1b8e93, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731307235728 2024-11-11T06:40:39,800 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81d01942301946b3b4e1fb20bea8dd6b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731307235728 2024-11-11T06:40:39,802 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb626be39004f76a4f3bedeb9e6898e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731307236903 2024-11-11T06:40:39,802 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b2d37eeb38044c3db23e7419e15977fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731307238118 2024-11-11T06:40:39,802 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52a1c13f9bf64c7ab0a451b303a6f49a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731307236903 2024-11-11T06:40:39,803 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d0fd2e8e53d444d90c0065e46533582, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731307238118 2024-11-11T06:40:39,811 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:39,815 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:39,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T06:40:39,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,815 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T06:40:39,816 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/7e6ca703aaa441dbb95bf6d831f69142 is 50, key is test_row_0/B:col10/1731307238118/Put/seqid=0 2024-11-11T06:40:39,821 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111cdd6e459aaaf49bc90c0d3599f515194_1afd384ae6d17851e72f641429f124f2 store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:39,823 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111cdd6e459aaaf49bc90c0d3599f515194_1afd384ae6d17851e72f641429f124f2, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:39,823 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cdd6e459aaaf49bc90c0d3599f515194_1afd384ae6d17851e72f641429f124f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:39,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742020_1196 (size=12915) 2024-11-11T06:40:39,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111147b5c3d02fbd4fea8ec34d1799757495_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307238814/Put/seqid=0 2024-11-11T06:40:39,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742022_1198 (size=12454) 2024-11-11T06:40:39,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742021_1197 (size=4469) 2024-11-11T06:40:39,833 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/7e6ca703aaa441dbb95bf6d831f69142 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/7e6ca703aaa441dbb95bf6d831f69142 2024-11-11T06:40:39,838 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/B of 1afd384ae6d17851e72f641429f124f2 into 7e6ca703aaa441dbb95bf6d831f69142(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:39,838 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:39,838 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/B, priority=13, startTime=1731307239798; duration=0sec 2024-11-11T06:40:39,838 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:39,838 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:B 2024-11-11T06:40:39,838 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:39,839 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:39,839 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/C is initiating minor compaction (all files) 2024-11-11T06:40:39,839 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/C in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:39,839 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ba819cebf008416c822f5ea51f09e264, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ee50d004851442d1b7a92b1afc35f2bd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/7afa3bc0ba974970b9353d3fde0de173] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=36.2 K 2024-11-11T06:40:39,840 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ba819cebf008416c822f5ea51f09e264, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731307235728 2024-11-11T06:40:39,840 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ee50d004851442d1b7a92b1afc35f2bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731307236903 2024-11-11T06:40:39,840 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7afa3bc0ba974970b9353d3fde0de173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731307238118 2024-11-11T06:40:39,847 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:39,848 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/2c985f1f30044112b6106030ad6ad52d is 50, key is test_row_0/C:col10/1731307238118/Put/seqid=0 2024-11-11T06:40:39,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742023_1199 (size=12915) 2024-11-11T06:40:39,882 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/2c985f1f30044112b6106030ad6ad52d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2c985f1f30044112b6106030ad6ad52d 2024-11-11T06:40:39,888 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/C of 1afd384ae6d17851e72f641429f124f2 into 2c985f1f30044112b6106030ad6ad52d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:39,888 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:39,889 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/C, priority=13, startTime=1731307239798; duration=0sec 2024-11-11T06:40:39,889 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:39,889 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:C 2024-11-11T06:40:39,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T06:40:39,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307299977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307299978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:39,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307299978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307300089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307300089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307300089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:40,233 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:40,234 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/099f473774df41b7921affe2a4fbe89e is 175, key is test_row_0/A:col10/1731307238118/Put/seqid=0 2024-11-11T06:40:40,237 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111147b5c3d02fbd4fea8ec34d1799757495_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111147b5c3d02fbd4fea8ec34d1799757495_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:40,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f4f4c9076f0346b1a5736ef1420e899f, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:40,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f4f4c9076f0346b1a5736ef1420e899f is 175, key is test_row_0/A:col10/1731307238814/Put/seqid=0 2024-11-11T06:40:40,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742024_1200 (size=31869) 2024-11-11T06:40:40,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742025_1201 (size=31255) 2024-11-11T06:40:40,258 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f4f4c9076f0346b1a5736ef1420e899f 2024-11-11T06:40:40,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/814fe573271c48d19bdc5b27b59f5b5d is 50, key is test_row_0/B:col10/1731307238814/Put/seqid=0 2024-11-11T06:40:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307300299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307300299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307300299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742026_1202 (size=12301) 2024-11-11T06:40:40,324 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/814fe573271c48d19bdc5b27b59f5b5d 2024-11-11T06:40:40,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/c1f3b7d7cbba4f798025970e5d29cac5 is 50, key is test_row_0/C:col10/1731307238814/Put/seqid=0 2024-11-11T06:40:40,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742027_1203 (size=12301) 2024-11-11T06:40:40,340 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/c1f3b7d7cbba4f798025970e5d29cac5 2024-11-11T06:40:40,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f4f4c9076f0346b1a5736ef1420e899f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f4f4c9076f0346b1a5736ef1420e899f 2024-11-11T06:40:40,348 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f4f4c9076f0346b1a5736ef1420e899f, entries=150, sequenceid=288, filesize=30.5 K 2024-11-11T06:40:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/814fe573271c48d19bdc5b27b59f5b5d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/814fe573271c48d19bdc5b27b59f5b5d 2024-11-11T06:40:40,353 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/814fe573271c48d19bdc5b27b59f5b5d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-11T06:40:40,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/c1f3b7d7cbba4f798025970e5d29cac5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/c1f3b7d7cbba4f798025970e5d29cac5 2024-11-11T06:40:40,360 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/c1f3b7d7cbba4f798025970e5d29cac5, entries=150, sequenceid=288, filesize=12.0 K 2024-11-11T06:40:40,361 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=93.93 KB/96180 for 1afd384ae6d17851e72f641429f124f2 in 546ms, sequenceid=288, compaction requested=false 2024-11-11T06:40:40,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:40,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:40,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T06:40:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T06:40:40,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T06:40:40,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-11-11T06:40:40,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0160 sec 2024-11-11T06:40:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T06:40:40,489 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:40,489 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T06:40:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T06:40:40,491 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:40,491 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:40,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T06:40:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:40,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-11T06:40:40,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116fea64e3df3b4646a7f04297332af853_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307240612/Put/seqid=0 2024-11-11T06:40:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742028_1204 (size=14994) 2024-11-11T06:40:40,625 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:40,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307300631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,634 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116fea64e3df3b4646a7f04297332af853_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116fea64e3df3b4646a7f04297332af853_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,635 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/7e0130a6f4cd4e3d8ede6fb6b69036ec, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:40,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/7e0130a6f4cd4e3d8ede6fb6b69036ec is 175, key is test_row_0/A:col10/1731307240612/Put/seqid=0 2024-11-11T06:40:40,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307300633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307300634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T06:40:40,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:40,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:40,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:40,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742029_1205 (size=39949) 2024-11-11T06:40:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,644 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/7e0130a6f4cd4e3d8ede6fb6b69036ec 2024-11-11T06:40:40,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/55974864ed94417593254da85f6f2817 is 50, key is test_row_0/B:col10/1731307240612/Put/seqid=0 2024-11-11T06:40:40,658 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/099f473774df41b7921affe2a4fbe89e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/099f473774df41b7921affe2a4fbe89e 2024-11-11T06:40:40,663 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/A of 1afd384ae6d17851e72f641429f124f2 into 099f473774df41b7921affe2a4fbe89e(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:40,663 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:40,663 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/A, priority=13, startTime=1731307239798; duration=0sec 2024-11-11T06:40:40,663 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:40,663 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:A 2024-11-11T06:40:40,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742030_1206 (size=12301) 2024-11-11T06:40:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307300739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307300749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307300749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T06:40:40,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:40,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:40,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:40,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T06:40:40,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T06:40:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:40,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:40,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307300949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307300959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307300959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/55974864ed94417593254da85f6f2817 2024-11-11T06:40:41,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/b5bb3709bf65487385bdc8af15b3a515 is 50, key is test_row_0/C:col10/1731307240612/Put/seqid=0 2024-11-11T06:40:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742031_1207 (size=12301) 2024-11-11T06:40:41,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T06:40:41,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:41,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T06:40:41,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T06:40:41,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:41,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307301259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307301269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307301269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T06:40:41,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:41,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:41,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/b5bb3709bf65487385bdc8af15b3a515 2024-11-11T06:40:41,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/7e0130a6f4cd4e3d8ede6fb6b69036ec as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/7e0130a6f4cd4e3d8ede6fb6b69036ec 2024-11-11T06:40:41,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/7e0130a6f4cd4e3d8ede6fb6b69036ec, entries=200, sequenceid=310, filesize=39.0 K 2024-11-11T06:40:41,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/55974864ed94417593254da85f6f2817 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/55974864ed94417593254da85f6f2817 2024-11-11T06:40:41,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/55974864ed94417593254da85f6f2817, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T06:40:41,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/b5bb3709bf65487385bdc8af15b3a515 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b5bb3709bf65487385bdc8af15b3a515 2024-11-11T06:40:41,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b5bb3709bf65487385bdc8af15b3a515, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T06:40:41,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1afd384ae6d17851e72f641429f124f2 in 902ms, sequenceid=310, compaction requested=true 2024-11-11T06:40:41,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:41,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:41,515 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:41,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:41,515 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:41,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:41,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:41,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:41,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:41,517 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:41,517 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:41,517 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/A is initiating minor compaction (all files) 2024-11-11T06:40:41,517 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/B is initiating minor compaction (all files) 2024-11-11T06:40:41,517 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/A in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,517 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/B in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,518 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/099f473774df41b7921affe2a4fbe89e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f4f4c9076f0346b1a5736ef1420e899f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/7e0130a6f4cd4e3d8ede6fb6b69036ec] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=100.7 K 2024-11-11T06:40:41,518 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/7e6ca703aaa441dbb95bf6d831f69142, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/814fe573271c48d19bdc5b27b59f5b5d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/55974864ed94417593254da85f6f2817] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=36.6 K 2024-11-11T06:40:41,518 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,518 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/099f473774df41b7921affe2a4fbe89e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f4f4c9076f0346b1a5736ef1420e899f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/7e0130a6f4cd4e3d8ede6fb6b69036ec] 2024-11-11T06:40:41,518 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e6ca703aaa441dbb95bf6d831f69142, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731307238118 2024-11-11T06:40:41,518 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 099f473774df41b7921affe2a4fbe89e, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731307238118 2024-11-11T06:40:41,519 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 814fe573271c48d19bdc5b27b59f5b5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731307238796 2024-11-11T06:40:41,519 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f4c9076f0346b1a5736ef1420e899f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731307238796 2024-11-11T06:40:41,520 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 55974864ed94417593254da85f6f2817, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731307239975 2024-11-11T06:40:41,520 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0130a6f4cd4e3d8ede6fb6b69036ec, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731307239975 2024-11-11T06:40:41,533 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:41,533 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/57ace3d764a14a2a8b1c7c5d5f988796 is 50, key is test_row_0/B:col10/1731307240612/Put/seqid=0 2024-11-11T06:40:41,539 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:41,553 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111336e539bd328414d844a27fe9816e777_1afd384ae6d17851e72f641429f124f2 store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:41,556 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111336e539bd328414d844a27fe9816e777_1afd384ae6d17851e72f641429f124f2, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:41,556 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111336e539bd328414d844a27fe9816e777_1afd384ae6d17851e72f641429f124f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:41,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T06:40:41,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,564 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T06:40:41,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742032_1208 (size=13017) 2024-11-11T06:40:41,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742033_1209 (size=4469) 2024-11-11T06:40:41,585 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#A#compaction#172 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:41,586 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/8a076c7f4fd446ae885815936a505f40 is 175, key is test_row_0/A:col10/1731307240612/Put/seqid=0 2024-11-11T06:40:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a154ff1b82ca4337848a6a24d0b91410_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307240632/Put/seqid=0 2024-11-11T06:40:41,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742034_1210 (size=31971) 2024-11-11T06:40:41,627 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/8a076c7f4fd446ae885815936a505f40 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8a076c7f4fd446ae885815936a505f40 2024-11-11T06:40:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T06:40:41,637 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/A of 1afd384ae6d17851e72f641429f124f2 into 8a076c7f4fd446ae885815936a505f40(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:41,637 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:41,637 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/A, priority=13, startTime=1731307241515; duration=0sec 2024-11-11T06:40:41,637 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:41,637 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:A 2024-11-11T06:40:41,637 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:40:41,639 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:40:41,639 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/C is initiating minor compaction (all files) 2024-11-11T06:40:41,639 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/C in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:41,639 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2c985f1f30044112b6106030ad6ad52d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/c1f3b7d7cbba4f798025970e5d29cac5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b5bb3709bf65487385bdc8af15b3a515] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=36.6 K 2024-11-11T06:40:41,641 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c985f1f30044112b6106030ad6ad52d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731307238118 2024-11-11T06:40:41,643 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f3b7d7cbba4f798025970e5d29cac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731307238796 2024-11-11T06:40:41,644 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5bb3709bf65487385bdc8af15b3a515, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731307239975 2024-11-11T06:40:41,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742035_1211 (size=12454) 2024-11-11T06:40:41,656 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:40:41,657 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/983a00ec3034489297c9e45b78830bb9 is 50, key is test_row_0/C:col10/1731307240612/Put/seqid=0 2024-11-11T06:40:41,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742036_1212 (size=13017) 2024-11-11T06:40:41,680 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/983a00ec3034489297c9e45b78830bb9 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/983a00ec3034489297c9e45b78830bb9 2024-11-11T06:40:41,686 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/C of 1afd384ae6d17851e72f641429f124f2 into 983a00ec3034489297c9e45b78830bb9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:41,686 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:41,686 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/C, priority=13, startTime=1731307241515; duration=0sec 2024-11-11T06:40:41,686 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:41,686 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:C 2024-11-11T06:40:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:41,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307301801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307301805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307301805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307301909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307301919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307301919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:41,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:41,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T06:40:41,971 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/57ace3d764a14a2a8b1c7c5d5f988796 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/57ace3d764a14a2a8b1c7c5d5f988796 2024-11-11T06:40:41,976 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/B of 1afd384ae6d17851e72f641429f124f2 into 57ace3d764a14a2a8b1c7c5d5f988796(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:41,976 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:41,976 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/B, priority=13, startTime=1731307241515; duration=0sec 2024-11-11T06:40:41,976 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:41,976 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:B 2024-11-11T06:40:42,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:42,059 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a154ff1b82ca4337848a6a24d0b91410_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a154ff1b82ca4337848a6a24d0b91410_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:42,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f31da61ae6e0491a9c8549a5be395bb7, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:42,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f31da61ae6e0491a9c8549a5be395bb7 is 175, key is test_row_0/A:col10/1731307240632/Put/seqid=0 2024-11-11T06:40:42,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742037_1213 (size=31255) 2024-11-11T06:40:42,073 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f31da61ae6e0491a9c8549a5be395bb7 2024-11-11T06:40:42,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/4e849b76f8cf4c2f9f9eb4c780b9d840 is 50, key is test_row_0/B:col10/1731307240632/Put/seqid=0 2024-11-11T06:40:42,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742038_1214 (size=12301) 2024-11-11T06:40:42,099 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/4e849b76f8cf4c2f9f9eb4c780b9d840 2024-11-11T06:40:42,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/832bbdc587374364a79b3617ff4873c6 is 50, key is test_row_0/C:col10/1731307240632/Put/seqid=0 2024-11-11T06:40:42,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307302120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742039_1215 (size=12301) 2024-11-11T06:40:42,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307302129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:42,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307302130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:42,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307302429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:42,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307302439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:42,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307302440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:42,530 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/832bbdc587374364a79b3617ff4873c6 2024-11-11T06:40:42,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/f31da61ae6e0491a9c8549a5be395bb7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f31da61ae6e0491a9c8549a5be395bb7 2024-11-11T06:40:42,540 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f31da61ae6e0491a9c8549a5be395bb7, entries=150, sequenceid=327, filesize=30.5 K 2024-11-11T06:40:42,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/4e849b76f8cf4c2f9f9eb4c780b9d840 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4e849b76f8cf4c2f9f9eb4c780b9d840 2024-11-11T06:40:42,548 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4e849b76f8cf4c2f9f9eb4c780b9d840, entries=150, sequenceid=327, filesize=12.0 K 2024-11-11T06:40:42,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/832bbdc587374364a79b3617ff4873c6 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/832bbdc587374364a79b3617ff4873c6 2024-11-11T06:40:42,558 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/832bbdc587374364a79b3617ff4873c6, entries=150, sequenceid=327, filesize=12.0 K 2024-11-11T06:40:42,559 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 1afd384ae6d17851e72f641429f124f2 in 995ms, sequenceid=327, compaction requested=false 2024-11-11T06:40:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T06:40:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T06:40:42,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T06:40:42,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0690 sec 2024-11-11T06:40:42,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0730 sec 2024-11-11T06:40:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T06:40:42,639 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:42,641 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-11T06:40:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T06:40:42,642 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:42,643 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:42,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T06:40:42,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T06:40:42,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:42,796 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T06:40:42,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114f053b343ebf4eaf9114ef22d91f5ba2_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307241804/Put/seqid=0 2024-11-11T06:40:42,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742040_1216 (size=12454) 2024-11-11T06:40:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:42,830 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114f053b343ebf4eaf9114ef22d91f5ba2_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114f053b343ebf4eaf9114ef22d91f5ba2_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:42,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/49fb045a6656436496591c1617975c3a, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/49fb045a6656436496591c1617975c3a is 175, key is test_row_0/A:col10/1731307241804/Put/seqid=0 2024-11-11T06:40:42,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742041_1217 (size=31255) 2024-11-11T06:40:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:42,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T06:40:42,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307302960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:42,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307302963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:42,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307302964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307303070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307303080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307303080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,248 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/49fb045a6656436496591c1617975c3a 2024-11-11T06:40:43,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/96b65bddffaf41f1b2b04f9003c3f021 is 50, key is test_row_0/B:col10/1731307241804/Put/seqid=0 2024-11-11T06:40:43,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742042_1218 (size=12301) 2024-11-11T06:40:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T06:40:43,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307303279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307303289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307303289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307303589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307303599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307303599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/96b65bddffaf41f1b2b04f9003c3f021 2024-11-11T06:40:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/3b1f22093cc744e38c31703e553559d5 is 50, key is test_row_0/C:col10/1731307241804/Put/seqid=0 2024-11-11T06:40:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742043_1219 (size=12301) 2024-11-11T06:40:43,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60634 deadline: 1731307303763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:43,769 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1afd384ae6d17851e72f641429f124f2 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T06:40:44,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307304099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:44,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307304109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:44,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307304109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:44,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/3b1f22093cc744e38c31703e553559d5 2024-11-11T06:40:44,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/49fb045a6656436496591c1617975c3a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/49fb045a6656436496591c1617975c3a 2024-11-11T06:40:44,159 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/49fb045a6656436496591c1617975c3a, entries=150, sequenceid=351, filesize=30.5 K 2024-11-11T06:40:44,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/96b65bddffaf41f1b2b04f9003c3f021 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/96b65bddffaf41f1b2b04f9003c3f021 2024-11-11T06:40:44,166 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/96b65bddffaf41f1b2b04f9003c3f021, entries=150, sequenceid=351, filesize=12.0 K 2024-11-11T06:40:44,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/3b1f22093cc744e38c31703e553559d5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/3b1f22093cc744e38c31703e553559d5 2024-11-11T06:40:44,171 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/3b1f22093cc744e38c31703e553559d5, entries=150, sequenceid=351, filesize=12.0 K 2024-11-11T06:40:44,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 1afd384ae6d17851e72f641429f124f2 in 1376ms, sequenceid=351, compaction requested=true 2024-11-11T06:40:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-11T06:40:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-11T06:40:44,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T06:40:44,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5300 sec 2024-11-11T06:40:44,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5340 sec 2024-11-11T06:40:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T06:40:44,789 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:44,789 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-11T06:40:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T06:40:44,791 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:44,791 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:44,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:44,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T06:40:44,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T06:40:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:44,944 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T06:40:44,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112688cd33242e43b1962e5bda495be4bb_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307242952/Put/seqid=0 2024-11-11T06:40:44,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742044_1220 (size=12454) 2024-11-11T06:40:44,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:44,971 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112688cd33242e43b1962e5bda495be4bb_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112688cd33242e43b1962e5bda495be4bb_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:44,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/da97fc50f8ce40c7af45daec86cc4f0b, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:44,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/da97fc50f8ce40c7af45daec86cc4f0b is 175, key is test_row_0/A:col10/1731307242952/Put/seqid=0 2024-11-11T06:40:44,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742045_1221 (size=31255) 2024-11-11T06:40:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T06:40:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:45,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. as already flushing 2024-11-11T06:40:45,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307305152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307305153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307305154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307305259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307305269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307305270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,377 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/da97fc50f8ce40c7af45daec86cc4f0b 2024-11-11T06:40:45,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/884c5c2455b045ff9a3c794fa86ec02c is 50, key is test_row_0/B:col10/1731307242952/Put/seqid=0 2024-11-11T06:40:45,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742046_1222 (size=12301) 2024-11-11T06:40:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T06:40:45,448 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-11T06:40:45,448 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:45,448 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:45,448 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:45,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:45,450 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-11T06:40:45,450 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:45,450 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:45,450 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:45,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:45,452 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-11T06:40:45,452 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:45,452 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:45,452 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:45,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:45,461 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-11T06:40:45,461 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:45,461 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:45,461 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:45,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307305469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307305479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307305479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60712 deadline: 1731307305779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60660 deadline: 1731307305790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60680 deadline: 1731307305790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:45,802 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/884c5c2455b045ff9a3c794fa86ec02c 2024-11-11T06:40:45,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/06d9622835f5478e9f3ce99a2a4e6c29 is 50, key is test_row_0/C:col10/1731307242952/Put/seqid=0 2024-11-11T06:40:45,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742047_1223 (size=12301) 2024-11-11T06:40:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T06:40:45,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60632 deadline: 1731307305979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:45,980 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1afd384ae6d17851e72f641429f124f2 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:40:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:40:46,220 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/06d9622835f5478e9f3ce99a2a4e6c29 2024-11-11T06:40:46,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/da97fc50f8ce40c7af45daec86cc4f0b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/da97fc50f8ce40c7af45daec86cc4f0b 2024-11-11T06:40:46,230 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/da97fc50f8ce40c7af45daec86cc4f0b, entries=150, sequenceid=366, filesize=30.5 K 2024-11-11T06:40:46,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/884c5c2455b045ff9a3c794fa86ec02c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/884c5c2455b045ff9a3c794fa86ec02c 2024-11-11T06:40:46,235 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/884c5c2455b045ff9a3c794fa86ec02c, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T06:40:46,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/06d9622835f5478e9f3ce99a2a4e6c29 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/06d9622835f5478e9f3ce99a2a4e6c29 2024-11-11T06:40:46,241 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/06d9622835f5478e9f3ce99a2a4e6c29, entries=150, sequenceid=366, filesize=12.0 K 2024-11-11T06:40:46,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1afd384ae6d17851e72f641429f124f2 in 1298ms, sequenceid=366, compaction requested=true 2024-11-11T06:40:46,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:46,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:46,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-11T06:40:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-11T06:40:46,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T06:40:46,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4530 sec 2024-11-11T06:40:46,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.4580 sec 2024-11-11T06:40:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:46,291 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:46,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T06:40:46,291 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-11T06:40:46,291 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:46,291 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:46,291 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:46,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:46,302 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-11T06:40:46,302 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:46,302 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:46,302 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:46,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:46,303 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-11T06:40:46,303 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:46,303 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:46,303 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:46,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:46,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117523f1aa182745ae948c5e92b08d9ef6_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307245144/Put/seqid=0 2024-11-11T06:40:46,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742048_1224 (size=12454) 2024-11-11T06:40:46,317 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:46,323 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117523f1aa182745ae948c5e92b08d9ef6_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117523f1aa182745ae948c5e92b08d9ef6_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:46,325 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/8718bd96a4b9494e94b3d13409987a73, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:46,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/8718bd96a4b9494e94b3d13409987a73 is 175, key is test_row_0/A:col10/1731307245144/Put/seqid=0 2024-11-11T06:40:46,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742049_1225 (size=31255) 2024-11-11T06:40:46,733 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/8718bd96a4b9494e94b3d13409987a73 2024-11-11T06:40:46,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/feeec53633824ef591fc8cda50f4aac2 is 50, key is test_row_0/B:col10/1731307245144/Put/seqid=0 2024-11-11T06:40:46,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742050_1226 (size=12301) 2024-11-11T06:40:46,914 DEBUG [regionserver/ce2cf0244921:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-11T06:40:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T06:40:46,940 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:47,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/feeec53633824ef591fc8cda50f4aac2 2024-11-11T06:40:47,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/fd66809709574465a6b0bd719f8a6f17 is 50, key is test_row_0/C:col10/1731307245144/Put/seqid=0 2024-11-11T06:40:47,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742051_1227 (size=12301) 2024-11-11T06:40:47,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/fd66809709574465a6b0bd719f8a6f17 2024-11-11T06:40:47,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/8718bd96a4b9494e94b3d13409987a73 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8718bd96a4b9494e94b3d13409987a73 2024-11-11T06:40:47,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8718bd96a4b9494e94b3d13409987a73, entries=150, sequenceid=388, filesize=30.5 K 2024-11-11T06:40:47,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/feeec53633824ef591fc8cda50f4aac2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/feeec53633824ef591fc8cda50f4aac2 2024-11-11T06:40:47,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/feeec53633824ef591fc8cda50f4aac2, entries=150, sequenceid=388, filesize=12.0 K 2024-11-11T06:40:47,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/fd66809709574465a6b0bd719f8a6f17 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/fd66809709574465a6b0bd719f8a6f17 2024-11-11T06:40:47,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/fd66809709574465a6b0bd719f8a6f17, entries=150, sequenceid=388, filesize=12.0 K 2024-11-11T06:40:47,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=13.42 KB/13740 for 1afd384ae6d17851e72f641429f124f2 in 1291ms, sequenceid=388, compaction requested=true 2024-11-11T06:40:47,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:47,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:40:47,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:40:47,583 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T06:40:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1afd384ae6d17851e72f641429f124f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:40:47,583 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T06:40:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:47,584 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156991 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T06:40:47,584 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T06:40:47,584 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/A is initiating minor compaction (all files) 2024-11-11T06:40:47,584 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/B is initiating minor compaction (all files) 2024-11-11T06:40:47,584 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/A in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:47,584 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/B in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:47,584 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/57ace3d764a14a2a8b1c7c5d5f988796, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4e849b76f8cf4c2f9f9eb4c780b9d840, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/96b65bddffaf41f1b2b04f9003c3f021, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/884c5c2455b045ff9a3c794fa86ec02c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/feeec53633824ef591fc8cda50f4aac2] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=60.8 K 2024-11-11T06:40:47,584 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8a076c7f4fd446ae885815936a505f40, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f31da61ae6e0491a9c8549a5be395bb7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/49fb045a6656436496591c1617975c3a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/da97fc50f8ce40c7af45daec86cc4f0b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8718bd96a4b9494e94b3d13409987a73] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=153.3 K 2024-11-11T06:40:47,584 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:47,584 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8a076c7f4fd446ae885815936a505f40, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f31da61ae6e0491a9c8549a5be395bb7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/49fb045a6656436496591c1617975c3a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/da97fc50f8ce40c7af45daec86cc4f0b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8718bd96a4b9494e94b3d13409987a73] 2024-11-11T06:40:47,585 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ace3d764a14a2a8b1c7c5d5f988796, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731307239975 2024-11-11T06:40:47,585 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a076c7f4fd446ae885815936a505f40, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731307239975 2024-11-11T06:40:47,585 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e849b76f8cf4c2f9f9eb4c780b9d840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731307240627 2024-11-11T06:40:47,585 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting f31da61ae6e0491a9c8549a5be395bb7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731307240627 2024-11-11T06:40:47,585 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 96b65bddffaf41f1b2b04f9003c3f021, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731307241798 2024-11-11T06:40:47,585 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49fb045a6656436496591c1617975c3a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731307241798 2024-11-11T06:40:47,586 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 884c5c2455b045ff9a3c794fa86ec02c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731307242952 2024-11-11T06:40:47,586 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting da97fc50f8ce40c7af45daec86cc4f0b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731307242952 2024-11-11T06:40:47,586 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting feeec53633824ef591fc8cda50f4aac2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731307245144 2024-11-11T06:40:47,586 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8718bd96a4b9494e94b3d13409987a73, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731307245144 2024-11-11T06:40:47,599 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:47,602 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#B#compaction#186 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:40:47,603 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/2081067bb3294c4da8788b425c159c04 is 50, key is test_row_0/B:col10/1731307245144/Put/seqid=0 2024-11-11T06:40:47,604 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111b6702189c15b49d8b79ae8e24c0e9eb6_1afd384ae6d17851e72f641429f124f2 store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:47,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742052_1228 (size=13187) 2024-11-11T06:40:47,636 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111b6702189c15b49d8b79ae8e24c0e9eb6_1afd384ae6d17851e72f641429f124f2, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:47,636 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b6702189c15b49d8b79ae8e24c0e9eb6_1afd384ae6d17851e72f641429f124f2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742053_1229 (size=4469) 2024-11-11T06:40:48,012 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/2081067bb3294c4da8788b425c159c04 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2081067bb3294c4da8788b425c159c04 2024-11-11T06:40:48,016 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/B of 1afd384ae6d17851e72f641429f124f2 into 2081067bb3294c4da8788b425c159c04(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:48,016 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:48,016 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/B, priority=11, startTime=1731307247582; duration=0sec 2024-11-11T06:40:48,016 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:40:48,016 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:B 2024-11-11T06:40:48,016 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T06:40:48,018 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T06:40:48,018 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1afd384ae6d17851e72f641429f124f2/C is initiating minor compaction (all files) 2024-11-11T06:40:48,018 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1afd384ae6d17851e72f641429f124f2/C in TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:48,018 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/983a00ec3034489297c9e45b78830bb9, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/832bbdc587374364a79b3617ff4873c6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/3b1f22093cc744e38c31703e553559d5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/06d9622835f5478e9f3ce99a2a4e6c29, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/fd66809709574465a6b0bd719f8a6f17] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp, totalSize=60.8 K 2024-11-11T06:40:48,019 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 983a00ec3034489297c9e45b78830bb9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731307239975 2024-11-11T06:40:48,019 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 832bbdc587374364a79b3617ff4873c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731307240627 2024-11-11T06:40:48,019 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b1f22093cc744e38c31703e553559d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731307241798 2024-11-11T06:40:48,020 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 06d9622835f5478e9f3ce99a2a4e6c29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731307242952 2024-11-11T06:40:48,020 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting fd66809709574465a6b0bd719f8a6f17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731307245144 2024-11-11T06:40:48,032 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#C#compaction#188 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:40:48,033 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/dea4d7991eb04f019e7c9b25050a7f4d is 50, key is test_row_0/C:col10/1731307245144/Put/seqid=0 2024-11-11T06:40:48,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742054_1230 (size=13187) 2024-11-11T06:40:48,041 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1afd384ae6d17851e72f641429f124f2#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:40:48,042 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/185ad6978d464247b7cc8815b98a5069 is 175, key is test_row_0/A:col10/1731307245144/Put/seqid=0 2024-11-11T06:40:48,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742055_1231 (size=32141) 2024-11-11T06:40:48,442 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/dea4d7991eb04f019e7c9b25050a7f4d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/dea4d7991eb04f019e7c9b25050a7f4d 2024-11-11T06:40:48,447 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/C of 1afd384ae6d17851e72f641429f124f2 into dea4d7991eb04f019e7c9b25050a7f4d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:48,448 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:48,448 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/C, priority=11, startTime=1731307247583; duration=0sec 2024-11-11T06:40:48,448 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:48,448 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:C 2024-11-11T06:40:48,451 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/185ad6978d464247b7cc8815b98a5069 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/185ad6978d464247b7cc8815b98a5069 2024-11-11T06:40:48,455 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 1afd384ae6d17851e72f641429f124f2/A of 1afd384ae6d17851e72f641429f124f2 into 185ad6978d464247b7cc8815b98a5069(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:40:48,455 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1afd384ae6d17851e72f641429f124f2: 2024-11-11T06:40:48,456 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., storeName=1afd384ae6d17851e72f641429f124f2/A, priority=11, startTime=1731307247582; duration=0sec 2024-11-11T06:40:48,456 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:40:48,456 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1afd384ae6d17851e72f641429f124f2:A 2024-11-11T06:40:53,845 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-11T06:40:53,845 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:53,845 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:53,846 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:53,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:55,992 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-11T06:40:55,992 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:40:55,992 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:55,993 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:55,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:55,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T06:40:55,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-11T06:40:55,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3156 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3271 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1458 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4374 rows 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1440 2024-11-11T06:40:55,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4317 rows 2024-11-11T06:40:55,994 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T06:40:55,995 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:40:55,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:55,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:55,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:40:55,997 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T06:40:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T06:40:56,047 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307256047"}]},"ts":"1731307256047"} 2024-11-11T06:40:56,050 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T06:40:56,050 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T06:40:56,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:40:56,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, UNASSIGN}] 2024-11-11T06:40:56,054 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, UNASSIGN 2024-11-11T06:40:56,054 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=1afd384ae6d17851e72f641429f124f2, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:56,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, UNASSIGN because future has completed 2024-11-11T06:40:56,055 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:40:56,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:40:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T06:40:56,209 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:56,209 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:40:56,210 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 1afd384ae6d17851e72f641429f124f2, disabling compactions & flushes 2024-11-11T06:40:56,210 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:56,210 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:56,210 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. after waiting 0 ms 2024-11-11T06:40:56,210 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:56,211 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 1afd384ae6d17851e72f641429f124f2 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T06:40:56,222 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0eaae2d9175469c9fdb2cb1dbf94deb_1afd384ae6d17851e72f641429f124f2 is 50, key is test_row_0/A:col10/1731307246300/Put/seqid=0 2024-11-11T06:40:56,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742056_1232 (size=9914) 2024-11-11T06:40:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T06:40:56,626 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:56,635 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0eaae2d9175469c9fdb2cb1dbf94deb_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0eaae2d9175469c9fdb2cb1dbf94deb_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:56,636 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/1a53a1a589d24e859e21c60114ab1d04, store: [table=TestAcidGuarantees family=A region=1afd384ae6d17851e72f641429f124f2] 2024-11-11T06:40:56,637 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/1a53a1a589d24e859e21c60114ab1d04 is 175, key is test_row_0/A:col10/1731307246300/Put/seqid=0 2024-11-11T06:40:56,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742057_1233 (size=22561) 2024-11-11T06:40:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T06:40:57,042 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/1a53a1a589d24e859e21c60114ab1d04 2024-11-11T06:40:57,049 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/72e27da75f4b48138e0a0ed2ae042316 is 50, key is test_row_0/B:col10/1731307246300/Put/seqid=0 2024-11-11T06:40:57,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742058_1234 (size=9857) 2024-11-11T06:40:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T06:40:57,453 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/72e27da75f4b48138e0a0ed2ae042316 2024-11-11T06:40:57,460 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/effe4145da4a413dbd8f2c3079dc64f8 is 50, key is test_row_0/C:col10/1731307246300/Put/seqid=0 2024-11-11T06:40:57,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742059_1235 (size=9857) 2024-11-11T06:40:57,864 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/effe4145da4a413dbd8f2c3079dc64f8 2024-11-11T06:40:57,870 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/A/1a53a1a589d24e859e21c60114ab1d04 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/1a53a1a589d24e859e21c60114ab1d04 2024-11-11T06:40:57,875 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/1a53a1a589d24e859e21c60114ab1d04, entries=100, sequenceid=398, filesize=22.0 K 2024-11-11T06:40:57,876 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/B/72e27da75f4b48138e0a0ed2ae042316 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/72e27da75f4b48138e0a0ed2ae042316 2024-11-11T06:40:57,880 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/72e27da75f4b48138e0a0ed2ae042316, entries=100, sequenceid=398, filesize=9.6 K 2024-11-11T06:40:57,881 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/.tmp/C/effe4145da4a413dbd8f2c3079dc64f8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/effe4145da4a413dbd8f2c3079dc64f8 2024-11-11T06:40:57,886 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/effe4145da4a413dbd8f2c3079dc64f8, entries=100, sequenceid=398, filesize=9.6 K 2024-11-11T06:40:57,887 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1afd384ae6d17851e72f641429f124f2 in 1677ms, sequenceid=398, compaction requested=false 2024-11-11T06:40:57,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/3b5cce1a29884c13aaad4ba900f61e70, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a1bd722f2672475699fb2608b7b040a8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/14d6958cf1ff48af95e4b170d21d8d6c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/015b3ee06a804e7da13b5546ed8dcbb3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a88539d28a0a4192aaa9d5ced88e979a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/59ff620a893d41c39421baa88f7c5336, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/68b0217dd62e4d1a9a8101f6a8f71a4b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/607e1ffd19924115bda4da52f7fa698f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/06f3d5d6210c499c99b9e1a83303e753, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/297f14fd8f16448baad0d5e2f9f1abfd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/61900b6df4fc405f926a404006dc1838, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f70515fdd6874bd0bc58d5031413a87e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/ccd2dc52dcd04f4da26609a6d06c33ec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/0e0520ace97945f1b59ebf206831572f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/2760f05cee9c4f26af4640aa22910d57, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/81d01942301946b3b4e1fb20bea8dd6b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/52a1c13f9bf64c7ab0a451b303a6f49a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/099f473774df41b7921affe2a4fbe89e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/4d0fd2e8e53d444d90c0065e46533582, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f4f4c9076f0346b1a5736ef1420e899f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/7e0130a6f4cd4e3d8ede6fb6b69036ec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8a076c7f4fd446ae885815936a505f40, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f31da61ae6e0491a9c8549a5be395bb7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/49fb045a6656436496591c1617975c3a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/da97fc50f8ce40c7af45daec86cc4f0b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8718bd96a4b9494e94b3d13409987a73] to archive 2024-11-11T06:40:57,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:40:57,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/3b5cce1a29884c13aaad4ba900f61e70 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/3b5cce1a29884c13aaad4ba900f61e70 2024-11-11T06:40:57,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a1bd722f2672475699fb2608b7b040a8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a1bd722f2672475699fb2608b7b040a8 2024-11-11T06:40:57,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/14d6958cf1ff48af95e4b170d21d8d6c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/14d6958cf1ff48af95e4b170d21d8d6c 2024-11-11T06:40:57,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/015b3ee06a804e7da13b5546ed8dcbb3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/015b3ee06a804e7da13b5546ed8dcbb3 2024-11-11T06:40:57,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a88539d28a0a4192aaa9d5ced88e979a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/a88539d28a0a4192aaa9d5ced88e979a 2024-11-11T06:40:57,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/59ff620a893d41c39421baa88f7c5336 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/59ff620a893d41c39421baa88f7c5336 2024-11-11T06:40:57,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/68b0217dd62e4d1a9a8101f6a8f71a4b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/68b0217dd62e4d1a9a8101f6a8f71a4b 2024-11-11T06:40:57,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/607e1ffd19924115bda4da52f7fa698f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/607e1ffd19924115bda4da52f7fa698f 2024-11-11T06:40:57,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/06f3d5d6210c499c99b9e1a83303e753 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/06f3d5d6210c499c99b9e1a83303e753 2024-11-11T06:40:57,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/297f14fd8f16448baad0d5e2f9f1abfd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/297f14fd8f16448baad0d5e2f9f1abfd 2024-11-11T06:40:57,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/61900b6df4fc405f926a404006dc1838 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/61900b6df4fc405f926a404006dc1838 2024-11-11T06:40:57,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f70515fdd6874bd0bc58d5031413a87e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f70515fdd6874bd0bc58d5031413a87e 2024-11-11T06:40:57,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/ccd2dc52dcd04f4da26609a6d06c33ec to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/ccd2dc52dcd04f4da26609a6d06c33ec 2024-11-11T06:40:57,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/0e0520ace97945f1b59ebf206831572f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/0e0520ace97945f1b59ebf206831572f 2024-11-11T06:40:57,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/2760f05cee9c4f26af4640aa22910d57 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/2760f05cee9c4f26af4640aa22910d57 2024-11-11T06:40:57,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/81d01942301946b3b4e1fb20bea8dd6b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/81d01942301946b3b4e1fb20bea8dd6b 2024-11-11T06:40:57,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/52a1c13f9bf64c7ab0a451b303a6f49a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/52a1c13f9bf64c7ab0a451b303a6f49a 2024-11-11T06:40:57,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/099f473774df41b7921affe2a4fbe89e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/099f473774df41b7921affe2a4fbe89e 2024-11-11T06:40:57,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/4d0fd2e8e53d444d90c0065e46533582 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/4d0fd2e8e53d444d90c0065e46533582 2024-11-11T06:40:57,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f4f4c9076f0346b1a5736ef1420e899f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f4f4c9076f0346b1a5736ef1420e899f 2024-11-11T06:40:57,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/7e0130a6f4cd4e3d8ede6fb6b69036ec to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/7e0130a6f4cd4e3d8ede6fb6b69036ec 2024-11-11T06:40:57,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8a076c7f4fd446ae885815936a505f40 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8a076c7f4fd446ae885815936a505f40 2024-11-11T06:40:57,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f31da61ae6e0491a9c8549a5be395bb7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/f31da61ae6e0491a9c8549a5be395bb7 2024-11-11T06:40:57,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/49fb045a6656436496591c1617975c3a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/49fb045a6656436496591c1617975c3a 2024-11-11T06:40:57,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/da97fc50f8ce40c7af45daec86cc4f0b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/da97fc50f8ce40c7af45daec86cc4f0b 2024-11-11T06:40:57,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8718bd96a4b9494e94b3d13409987a73 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/8718bd96a4b9494e94b3d13409987a73 2024-11-11T06:40:57,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5f1d108e6535437ea9f29757d7c103b5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/838f78f627ee4deba9489481a85c5a12, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2fcc6d541b1f4fd1acea26f674145de2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c410127a6a244a3fbef73376eb6dcf19, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4ec1cf1e010642d48841a0ed9fc6b4cc, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/749c9dfd1bb2464983e77d2abb143ff3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/51e5a9776f2a4de79f432420c1740911, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c1535e9912e147acb1147af2d85e4455, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/37208ef2832249cbaadac0176a045a61, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2d3adae3281b4ccfa99bee4ae968705d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c5092c787c93439f95adc60510bb8a20, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/936767186b7a4f349c8493eef7c0f834, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5cc3b4304c05463790231104ec66c701, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/78998379c5374c4fa2eaf1cea947458e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/11225b66deac480aacaf8fec8e1b8e93, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/119da708bd9042a8bdbfe797e81afb1a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/1bb626be39004f76a4f3bedeb9e6898e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/7e6ca703aaa441dbb95bf6d831f69142, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/b2d37eeb38044c3db23e7419e15977fb, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/814fe573271c48d19bdc5b27b59f5b5d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/57ace3d764a14a2a8b1c7c5d5f988796, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/55974864ed94417593254da85f6f2817, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4e849b76f8cf4c2f9f9eb4c780b9d840, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/96b65bddffaf41f1b2b04f9003c3f021, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/884c5c2455b045ff9a3c794fa86ec02c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/feeec53633824ef591fc8cda50f4aac2] to archive 2024-11-11T06:40:57,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:40:57,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5f1d108e6535437ea9f29757d7c103b5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5f1d108e6535437ea9f29757d7c103b5 2024-11-11T06:40:57,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/838f78f627ee4deba9489481a85c5a12 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/838f78f627ee4deba9489481a85c5a12 2024-11-11T06:40:57,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2fcc6d541b1f4fd1acea26f674145de2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2fcc6d541b1f4fd1acea26f674145de2 2024-11-11T06:40:57,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c410127a6a244a3fbef73376eb6dcf19 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c410127a6a244a3fbef73376eb6dcf19 2024-11-11T06:40:57,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4ec1cf1e010642d48841a0ed9fc6b4cc to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4ec1cf1e010642d48841a0ed9fc6b4cc 2024-11-11T06:40:57,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/749c9dfd1bb2464983e77d2abb143ff3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/749c9dfd1bb2464983e77d2abb143ff3 2024-11-11T06:40:57,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/51e5a9776f2a4de79f432420c1740911 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/51e5a9776f2a4de79f432420c1740911 2024-11-11T06:40:57,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c1535e9912e147acb1147af2d85e4455 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c1535e9912e147acb1147af2d85e4455 2024-11-11T06:40:57,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/37208ef2832249cbaadac0176a045a61 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/37208ef2832249cbaadac0176a045a61 2024-11-11T06:40:57,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2d3adae3281b4ccfa99bee4ae968705d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2d3adae3281b4ccfa99bee4ae968705d 2024-11-11T06:40:57,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c5092c787c93439f95adc60510bb8a20 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/c5092c787c93439f95adc60510bb8a20 2024-11-11T06:40:57,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/936767186b7a4f349c8493eef7c0f834 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/936767186b7a4f349c8493eef7c0f834 2024-11-11T06:40:57,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5cc3b4304c05463790231104ec66c701 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/5cc3b4304c05463790231104ec66c701 2024-11-11T06:40:57,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/78998379c5374c4fa2eaf1cea947458e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/78998379c5374c4fa2eaf1cea947458e 2024-11-11T06:40:57,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/11225b66deac480aacaf8fec8e1b8e93 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/11225b66deac480aacaf8fec8e1b8e93 2024-11-11T06:40:57,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/119da708bd9042a8bdbfe797e81afb1a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/119da708bd9042a8bdbfe797e81afb1a 2024-11-11T06:40:57,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/1bb626be39004f76a4f3bedeb9e6898e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/1bb626be39004f76a4f3bedeb9e6898e 2024-11-11T06:40:57,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/7e6ca703aaa441dbb95bf6d831f69142 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/7e6ca703aaa441dbb95bf6d831f69142 2024-11-11T06:40:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/b2d37eeb38044c3db23e7419e15977fb to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/b2d37eeb38044c3db23e7419e15977fb 2024-11-11T06:40:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/814fe573271c48d19bdc5b27b59f5b5d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/814fe573271c48d19bdc5b27b59f5b5d 2024-11-11T06:40:57,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/57ace3d764a14a2a8b1c7c5d5f988796 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/57ace3d764a14a2a8b1c7c5d5f988796 2024-11-11T06:40:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/55974864ed94417593254da85f6f2817 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/55974864ed94417593254da85f6f2817 2024-11-11T06:40:57,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4e849b76f8cf4c2f9f9eb4c780b9d840 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/4e849b76f8cf4c2f9f9eb4c780b9d840 2024-11-11T06:40:57,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/96b65bddffaf41f1b2b04f9003c3f021 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/96b65bddffaf41f1b2b04f9003c3f021 2024-11-11T06:40:57,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/884c5c2455b045ff9a3c794fa86ec02c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/884c5c2455b045ff9a3c794fa86ec02c 2024-11-11T06:40:57,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/feeec53633824ef591fc8cda50f4aac2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/feeec53633824ef591fc8cda50f4aac2 2024-11-11T06:40:57,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/a4b4f228941e4ba7b36626cff87b9a42, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6b072298643346d3b902e88c5fcc5122, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/de3c2b0ea3be4eacba4df55c28737b42, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/be6ea5ca258f4394bf64d196c533bc41, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b6f05480f0554bad9f4fb598ba10a4e2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6616c0c0bf6446d6b0c4ac63688f5553, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6851a27fa3914ee189a4a43a002d3130, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/0f4775c9f09446fca561e1f18a7b8840, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2651d1dae1a04626b7cc832843e310aa, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/d821e76e05434b21a6f928ab557fa098, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/9769972335db43e4b1000fcd6cd2e9dd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/620041b039224d33aa25c258ae7028b4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/18f15c0680514d019b6f3c664f562de3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/8a7d1c84811b43d297d0035d96a311a1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ba819cebf008416c822f5ea51f09e264, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/4577e9056cb744f0bff512a44dbc6848, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ee50d004851442d1b7a92b1afc35f2bd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2c985f1f30044112b6106030ad6ad52d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/7afa3bc0ba974970b9353d3fde0de173, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/c1f3b7d7cbba4f798025970e5d29cac5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/983a00ec3034489297c9e45b78830bb9, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b5bb3709bf65487385bdc8af15b3a515, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/832bbdc587374364a79b3617ff4873c6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/3b1f22093cc744e38c31703e553559d5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/06d9622835f5478e9f3ce99a2a4e6c29, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/fd66809709574465a6b0bd719f8a6f17] to archive 2024-11-11T06:40:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:40:57,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/a4b4f228941e4ba7b36626cff87b9a42 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/a4b4f228941e4ba7b36626cff87b9a42 2024-11-11T06:40:57,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6b072298643346d3b902e88c5fcc5122 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6b072298643346d3b902e88c5fcc5122 2024-11-11T06:40:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/de3c2b0ea3be4eacba4df55c28737b42 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/de3c2b0ea3be4eacba4df55c28737b42 2024-11-11T06:40:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/be6ea5ca258f4394bf64d196c533bc41 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/be6ea5ca258f4394bf64d196c533bc41 2024-11-11T06:40:57,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b6f05480f0554bad9f4fb598ba10a4e2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b6f05480f0554bad9f4fb598ba10a4e2 2024-11-11T06:40:57,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6616c0c0bf6446d6b0c4ac63688f5553 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6616c0c0bf6446d6b0c4ac63688f5553 2024-11-11T06:40:57,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6851a27fa3914ee189a4a43a002d3130 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/6851a27fa3914ee189a4a43a002d3130 2024-11-11T06:40:57,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/0f4775c9f09446fca561e1f18a7b8840 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/0f4775c9f09446fca561e1f18a7b8840 2024-11-11T06:40:57,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2651d1dae1a04626b7cc832843e310aa to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2651d1dae1a04626b7cc832843e310aa 2024-11-11T06:40:57,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/d821e76e05434b21a6f928ab557fa098 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/d821e76e05434b21a6f928ab557fa098 2024-11-11T06:40:57,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/9769972335db43e4b1000fcd6cd2e9dd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/9769972335db43e4b1000fcd6cd2e9dd 2024-11-11T06:40:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/620041b039224d33aa25c258ae7028b4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/620041b039224d33aa25c258ae7028b4 2024-11-11T06:40:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/18f15c0680514d019b6f3c664f562de3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/18f15c0680514d019b6f3c664f562de3 2024-11-11T06:40:57,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/8a7d1c84811b43d297d0035d96a311a1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/8a7d1c84811b43d297d0035d96a311a1 2024-11-11T06:40:57,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ba819cebf008416c822f5ea51f09e264 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ba819cebf008416c822f5ea51f09e264 2024-11-11T06:40:57,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/4577e9056cb744f0bff512a44dbc6848 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/4577e9056cb744f0bff512a44dbc6848 2024-11-11T06:40:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ee50d004851442d1b7a92b1afc35f2bd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/ee50d004851442d1b7a92b1afc35f2bd 2024-11-11T06:40:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2c985f1f30044112b6106030ad6ad52d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/2c985f1f30044112b6106030ad6ad52d 2024-11-11T06:40:57,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/7afa3bc0ba974970b9353d3fde0de173 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/7afa3bc0ba974970b9353d3fde0de173 2024-11-11T06:40:57,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/c1f3b7d7cbba4f798025970e5d29cac5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/c1f3b7d7cbba4f798025970e5d29cac5 2024-11-11T06:40:57,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/983a00ec3034489297c9e45b78830bb9 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/983a00ec3034489297c9e45b78830bb9 2024-11-11T06:40:57,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b5bb3709bf65487385bdc8af15b3a515 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/b5bb3709bf65487385bdc8af15b3a515 2024-11-11T06:40:57,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/832bbdc587374364a79b3617ff4873c6 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/832bbdc587374364a79b3617ff4873c6 2024-11-11T06:40:57,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/3b1f22093cc744e38c31703e553559d5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/3b1f22093cc744e38c31703e553559d5 2024-11-11T06:40:57,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/06d9622835f5478e9f3ce99a2a4e6c29 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/06d9622835f5478e9f3ce99a2a4e6c29 2024-11-11T06:40:57,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/fd66809709574465a6b0bd719f8a6f17 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/fd66809709574465a6b0bd719f8a6f17 2024-11-11T06:40:57,982 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/recovered.edits/401.seqid, newMaxSeqId=401, maxSeqId=4 2024-11-11T06:40:57,982 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. 2024-11-11T06:40:57,982 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 1afd384ae6d17851e72f641429f124f2: Waiting for close lock at 1731307256209Running coprocessor pre-close hooks at 1731307256209Disabling compacts and flushes for region at 1731307256209Disabling writes for close at 1731307256210 (+1 ms)Obtaining lock to block concurrent updates at 1731307256211 (+1 ms)Preparing flush snapshotting stores in 1afd384ae6d17851e72f641429f124f2 at 1731307256211Finished memstore snapshotting TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731307256212 (+1 ms)Flushing stores of TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2. at 1731307256212Flushing 1afd384ae6d17851e72f641429f124f2/A: creating writer at 1731307256212Flushing mob file 1afd384ae6d17851e72f641429f124f2/A: appending metadata at 1731307256221 (+9 ms)Flushing mob file 1afd384ae6d17851e72f641429f124f2/A: closing flushed file at 1731307256221Flushing 1afd384ae6d17851e72f641429f124f2/A: appending metadata at 1731307256636 (+415 ms)Flushing 1afd384ae6d17851e72f641429f124f2/A: closing flushed file at 1731307256636Flushing 1afd384ae6d17851e72f641429f124f2/B: creating writer at 1731307257046 (+410 ms)Flushing 1afd384ae6d17851e72f641429f124f2/B: appending metadata at 1731307257049 (+3 ms)Flushing 1afd384ae6d17851e72f641429f124f2/B: closing flushed file at 1731307257049Flushing 1afd384ae6d17851e72f641429f124f2/C: creating writer at 1731307257457 (+408 ms)Flushing 1afd384ae6d17851e72f641429f124f2/C: appending metadata at 1731307257459 (+2 ms)Flushing 1afd384ae6d17851e72f641429f124f2/C: closing flushed file at 1731307257459Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c1b9034: reopening flushed file at 1731307257869 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64eb7b70: reopening flushed file at 1731307257875 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45613d82: reopening flushed file at 1731307257881 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1afd384ae6d17851e72f641429f124f2 in 1677ms, sequenceid=398, compaction requested=false at 1731307257887 (+6 ms)Writing region close event to WAL at 1731307257978 (+91 ms)Running coprocessor post-close hooks at 1731307257982 (+4 ms)Closed at 1731307257982 2024-11-11T06:40:57,984 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:57,985 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=1afd384ae6d17851e72f641429f124f2, regionState=CLOSED 2024-11-11T06:40:57,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:40:57,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T06:40:57,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 1afd384ae6d17851e72f641429f124f2, server=ce2cf0244921,35807,1731307183669 in 1.9310 sec 2024-11-11T06:40:57,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-11T06:40:57,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1afd384ae6d17851e72f641429f124f2, UNASSIGN in 1.9350 sec 2024-11-11T06:40:57,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T06:40:57,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9390 sec 2024-11-11T06:40:57,993 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307257993"}]},"ts":"1731307257993"} 2024-11-11T06:40:57,994 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T06:40:57,994 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T06:40:57,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9980 sec 2024-11-11T06:40:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T06:40:58,189 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:58,189 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T06:40:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:58,191 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T06:40:58,192 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:58,195 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,197 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/recovered.edits] 2024-11-11T06:40:58,199 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/185ad6978d464247b7cc8815b98a5069 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/185ad6978d464247b7cc8815b98a5069 2024-11-11T06:40:58,201 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/1a53a1a589d24e859e21c60114ab1d04 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/A/1a53a1a589d24e859e21c60114ab1d04 2024-11-11T06:40:58,203 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2081067bb3294c4da8788b425c159c04 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/2081067bb3294c4da8788b425c159c04 2024-11-11T06:40:58,204 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/72e27da75f4b48138e0a0ed2ae042316 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/B/72e27da75f4b48138e0a0ed2ae042316 2024-11-11T06:40:58,206 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/dea4d7991eb04f019e7c9b25050a7f4d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/dea4d7991eb04f019e7c9b25050a7f4d 2024-11-11T06:40:58,207 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/effe4145da4a413dbd8f2c3079dc64f8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/C/effe4145da4a413dbd8f2c3079dc64f8 2024-11-11T06:40:58,210 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/recovered.edits/401.seqid to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2/recovered.edits/401.seqid 2024-11-11T06:40:58,210 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,211 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T06:40:58,211 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T06:40:58,212 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T06:40:58,215 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110f52f40dcbc453eac84c03130588440_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110f52f40dcbc453eac84c03130588440_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,216 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117a07caa417642599d1757da5e9dd0ab_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117a07caa417642599d1757da5e9dd0ab_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,217 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112688cd33242e43b1962e5bda495be4bb_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112688cd33242e43b1962e5bda495be4bb_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,218 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111134af525fc6ac43cd9ff2e932ea52116c_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111134af525fc6ac43cd9ff2e932ea52116c_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,219 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111147b5c3d02fbd4fea8ec34d1799757495_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111147b5c3d02fbd4fea8ec34d1799757495_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,220 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114f053b343ebf4eaf9114ef22d91f5ba2_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114f053b343ebf4eaf9114ef22d91f5ba2_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,221 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f7052a612ab469191c46e082ef5688d_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f7052a612ab469191c46e082ef5688d_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,222 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f89238dc8fc4640992ce3642d17f268_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115f89238dc8fc4640992ce3642d17f268_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,223 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116fea64e3df3b4646a7f04297332af853_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116fea64e3df3b4646a7f04297332af853_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,224 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117523f1aa182745ae948c5e92b08d9ef6_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117523f1aa182745ae948c5e92b08d9ef6_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,226 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117711ce0f11c248c88b7b52e30569c0ff_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117711ce0f11c248c88b7b52e30569c0ff_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,227 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118fd6a6115ca544da89b0a66897ad0856_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118fd6a6115ca544da89b0a66897ad0856_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,228 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193494a067d7d41d4b1c7d81ae9931084_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111193494a067d7d41d4b1c7d81ae9931084_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,229 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119d59ea676cbc49d6a64f2537a146f105_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119d59ea676cbc49d6a64f2537a146f105_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,230 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a154ff1b82ca4337848a6a24d0b91410_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a154ff1b82ca4337848a6a24d0b91410_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,231 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a72079110be44992acdb9ba91793c5f2_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a72079110be44992acdb9ba91793c5f2_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,233 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b0c8cfde41f84707b36b027b73a88a45_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b0c8cfde41f84707b36b027b73a88a45_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,234 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b11d11acb06b4b90a88b6fa4c10d3ee1_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b11d11acb06b4b90a88b6fa4c10d3ee1_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,235 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ceed39f477554ceb8a03f8848bc24e84_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ceed39f477554ceb8a03f8848bc24e84_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,237 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0eaae2d9175469c9fdb2cb1dbf94deb_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0eaae2d9175469c9fdb2cb1dbf94deb_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,238 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4e5aebfcc1c488f88bc683e2c0755bc_1afd384ae6d17851e72f641429f124f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4e5aebfcc1c488f88bc683e2c0755bc_1afd384ae6d17851e72f641429f124f2 2024-11-11T06:40:58,239 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T06:40:58,240 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:58,243 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T06:40:58,245 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T06:40:58,246 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:58,246 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T06:40:58,246 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731307258246"}]},"ts":"9223372036854775807"} 2024-11-11T06:40:58,248 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T06:40:58,248 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1afd384ae6d17851e72f641429f124f2, NAME => 'TestAcidGuarantees,,1731307222273.1afd384ae6d17851e72f641429f124f2.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T06:40:58,248 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T06:40:58,248 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731307258248"}]},"ts":"9223372036854775807"} 2024-11-11T06:40:58,250 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T06:40:58,251 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:58,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-11T06:40:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T06:40:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T06:40:58,300 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:58,308 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=479 (was 427) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11064 (was 11709) 2024-11-11T06:40:58,316 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=479, ProcessCount=11, AvailableMemoryMB=11063 2024-11-11T06:40:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:40:58,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:40:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T06:40:58,320 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T06:40:58,320 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:58,320 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-11T06:40:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-11T06:40:58,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T06:40:58,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742060_1236 (size=959) 2024-11-11T06:40:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-11T06:40:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-11T06:40:58,730 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 2024-11-11T06:40:58,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742061_1237 (size=53) 2024-11-11T06:40:58,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:40:58,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d55fa98fb98bf91a607d5c6eb8d4ba7d, disabling compactions & flushes 2024-11-11T06:40:58,738 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:58,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:58,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. after waiting 0 ms 2024-11-11T06:40:58,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:58,738 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:58,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: Waiting for close lock at 1731307258738Disabling compacts and flushes for region at 1731307258738Disabling writes for close at 1731307258738Writing region close event to WAL at 1731307258738Closed at 1731307258738 2024-11-11T06:40:58,740 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T06:40:58,740 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731307258740"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731307258740"}]},"ts":"1731307258740"} 2024-11-11T06:40:58,742 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T06:40:58,743 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T06:40:58,743 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307258743"}]},"ts":"1731307258743"} 2024-11-11T06:40:58,745 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T06:40:58,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, ASSIGN}] 2024-11-11T06:40:58,747 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, ASSIGN 2024-11-11T06:40:58,748 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, ASSIGN; state=OFFLINE, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=false 2024-11-11T06:40:58,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=d55fa98fb98bf91a607d5c6eb8d4ba7d, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:58,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, ASSIGN because future has completed 2024-11-11T06:40:58,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:40:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-11T06:40:59,055 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,055 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:40:59,056 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,056 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:40:59,056 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,056 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,057 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,058 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d55fa98fb98bf91a607d5c6eb8d4ba7d columnFamilyName A 2024-11-11T06:40:59,058 DEBUG [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:59,059 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] regionserver.HStore(327): Store=d55fa98fb98bf91a607d5c6eb8d4ba7d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:59,059 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,061 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d55fa98fb98bf91a607d5c6eb8d4ba7d columnFamilyName B 2024-11-11T06:40:59,061 DEBUG [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:59,061 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] regionserver.HStore(327): Store=d55fa98fb98bf91a607d5c6eb8d4ba7d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:59,061 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,062 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d55fa98fb98bf91a607d5c6eb8d4ba7d columnFamilyName C 2024-11-11T06:40:59,062 DEBUG [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:40:59,063 INFO [StoreOpener-d55fa98fb98bf91a607d5c6eb8d4ba7d-1 {}] regionserver.HStore(327): Store=d55fa98fb98bf91a607d5c6eb8d4ba7d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:40:59,063 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,064 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,064 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,065 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,065 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,066 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:40:59,067 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,069 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T06:40:59,069 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened d55fa98fb98bf91a607d5c6eb8d4ba7d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61842315, jitterRate=-0.07847769558429718}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:40:59,069 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,070 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: Running coprocessor pre-open hook at 1731307259056Writing region info on filesystem at 1731307259056Initializing all the Stores at 1731307259057 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307259057Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307259057Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307259057Cleaning up temporary data from old regions at 1731307259065 (+8 ms)Running coprocessor post-open hooks at 1731307259069 (+4 ms)Region opened successfully at 1731307259070 (+1 ms) 2024-11-11T06:40:59,071 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., pid=63, masterSystemTime=1731307259052 2024-11-11T06:40:59,073 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,073 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,073 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=d55fa98fb98bf91a607d5c6eb8d4ba7d, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:40:59,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T06:40:59,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 in 176 msec 2024-11-11T06:40:59,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-11T06:40:59,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, ASSIGN in 332 msec 2024-11-11T06:40:59,080 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T06:40:59,080 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307259080"}]},"ts":"1731307259080"} 2024-11-11T06:40:59,082 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T06:40:59,083 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T06:40:59,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 765 msec 2024-11-11T06:40:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-11T06:40:59,460 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:40:59,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,464 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e768b4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,467 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168cf949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,469 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,473 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T06:40:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,40845,1731307182847 2024-11-11T06:40:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b4ed6b0 2024-11-11T06:40:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T06:40:59,475 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T06:40:59,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,478 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1899d6fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,480 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117315dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,482 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d02280e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,483 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b8a44a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,485 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cdfba73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,487 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8adf2df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,490 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a3f60e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,492 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0603af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,495 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc94f3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,496 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc77777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,499 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77eef224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a03e73e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,503 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596ccfe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,505 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108a9b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,507 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b44b495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,509 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76989242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,511 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbdaabe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,512 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5bd06d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:40:59,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:40:59,515 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:40:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:40:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:40:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4546dc0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:40:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:40:59,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:40:59,517 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:40:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e42789, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:40:59,518 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,518 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,518 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,518 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,518 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:40:59,518 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,519 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,519 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-11T06:40:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T06:40:59,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:40:59,522 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:40:59,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:40:59,523 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,527 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,529 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:40:59,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:40:59,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:40:59,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:40:59,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:40:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:40:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:40:59,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:40:59,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/2739718129ea411a9893f35e8939ee40 is 50, key is test_row_0/A:col10/1731307259534/Put/seqid=0 2024-11-11T06:40:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307319578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307319578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307319580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307319584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307319586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742062_1238 (size=14341) 2024-11-11T06:40:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T06:40:59,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:40:59,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:40:59,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:59,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:59,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307319689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307319689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307319699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307319699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307319699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:40:59,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:40:59,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:59,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T06:40:59,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307319899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307319899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307319909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307319909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:40:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307319909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:40:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:40:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:40:59,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:40:59,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:40:59,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:40:59,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:59,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:40:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/2739718129ea411a9893f35e8939ee40 2024-11-11T06:41:00,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/ab86d38840ae4639b8605ce343793e66 is 50, key is test_row_0/B:col10/1731307259534/Put/seqid=0 2024-11-11T06:41:00,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742063_1239 (size=12001) 2024-11-11T06:41:00,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/ab86d38840ae4639b8605ce343793e66 2024-11-11T06:41:00,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T06:41:00,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/5484da26a67c4173b2ea6a54b5d1ba36 is 50, key is test_row_0/C:col10/1731307259534/Put/seqid=0 2024-11-11T06:41:00,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742064_1240 (size=12001) 2024-11-11T06:41:00,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307320209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307320209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307320219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307320219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307320220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:00,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/5484da26a67c4173b2ea6a54b5d1ba36 2024-11-11T06:41:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/2739718129ea411a9893f35e8939ee40 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2739718129ea411a9893f35e8939ee40 2024-11-11T06:41:00,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2739718129ea411a9893f35e8939ee40, entries=200, sequenceid=13, filesize=14.0 K 2024-11-11T06:41:00,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/ab86d38840ae4639b8605ce343793e66 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/ab86d38840ae4639b8605ce343793e66 2024-11-11T06:41:00,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/ab86d38840ae4639b8605ce343793e66, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T06:41:00,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:00,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/5484da26a67c4173b2ea6a54b5d1ba36 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5484da26a67c4173b2ea6a54b5d1ba36 2024-11-11T06:41:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5484da26a67c4173b2ea6a54b5d1ba36, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T06:41:00,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1069ms, sequenceid=13, compaction requested=false 2024-11-11T06:41:00,603 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T06:41:00,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T06:41:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:00,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:00,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T06:41:00,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/63de424bfbb940c5bee59f1a158ba400 is 50, key is test_row_0/A:col10/1731307260720/Put/seqid=0 2024-11-11T06:41:00,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307320729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307320729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307320729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307320730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307320732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742065_1241 (size=12001) 2024-11-11T06:41:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307320839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307320839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:00,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:00,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:00,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307321049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307321050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/63de424bfbb940c5bee59f1a158ba400 2024-11-11T06:41:01,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8 is 50, key is test_row_0/B:col10/1731307260720/Put/seqid=0 2024-11-11T06:41:01,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742066_1242 (size=12001) 2024-11-11T06:41:01,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307321359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307321359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:01,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:01,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:01,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8 2024-11-11T06:41:01,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/a816080ed9f54ac78b0e91e981737d1a is 50, key is test_row_0/C:col10/1731307260720/Put/seqid=0 2024-11-11T06:41:01,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742067_1243 (size=12001) 2024-11-11T06:41:01,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/a816080ed9f54ac78b0e91e981737d1a 2024-11-11T06:41:01,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/63de424bfbb940c5bee59f1a158ba400 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/63de424bfbb940c5bee59f1a158ba400 2024-11-11T06:41:01,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/63de424bfbb940c5bee59f1a158ba400, entries=150, sequenceid=39, filesize=11.7 K 2024-11-11T06:41:01,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8 2024-11-11T06:41:01,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8, entries=150, sequenceid=39, filesize=11.7 K 2024-11-11T06:41:01,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/a816080ed9f54ac78b0e91e981737d1a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a816080ed9f54ac78b0e91e981737d1a 2024-11-11T06:41:01,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a816080ed9f54ac78b0e91e981737d1a, entries=150, sequenceid=39, filesize=11.7 K 2024-11-11T06:41:01,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 924ms, sequenceid=39, compaction requested=false 2024-11-11T06:41:01,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T06:41:01,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T06:41:01,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:01,672 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T06:41:01,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/7f2b21d55f0e4c5dbe509add244242ec is 50, key is test_row_0/A:col10/1731307260723/Put/seqid=0 2024-11-11T06:41:01,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742068_1244 (size=12001) 2024-11-11T06:41:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:01,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:01,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307321820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307321825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307321829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307321870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307321869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307321939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307321939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307321939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,084 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/7f2b21d55f0e4c5dbe509add244242ec 2024-11-11T06:41:02,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/6b3a9554c7fd45d39d66621c71a8ab7b is 50, key is test_row_0/B:col10/1731307260723/Put/seqid=0 2024-11-11T06:41:02,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742069_1245 (size=12001) 2024-11-11T06:41:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307322149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307322149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307322149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307322459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307322459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307322459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,528 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/6b3a9554c7fd45d39d66621c71a8ab7b 2024-11-11T06:41:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/2317c056220340da89ffab8b64ab953b is 50, key is test_row_0/C:col10/1731307260723/Put/seqid=0 2024-11-11T06:41:02,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742070_1246 (size=12001) 2024-11-11T06:41:02,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/2317c056220340da89ffab8b64ab953b 2024-11-11T06:41:02,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/7f2b21d55f0e4c5dbe509add244242ec as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/7f2b21d55f0e4c5dbe509add244242ec 2024-11-11T06:41:02,626 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/7f2b21d55f0e4c5dbe509add244242ec, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T06:41:02,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/6b3a9554c7fd45d39d66621c71a8ab7b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6b3a9554c7fd45d39d66621c71a8ab7b 2024-11-11T06:41:02,632 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6b3a9554c7fd45d39d66621c71a8ab7b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T06:41:02,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/2317c056220340da89ffab8b64ab953b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/2317c056220340da89ffab8b64ab953b 2024-11-11T06:41:02,643 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/2317c056220340da89ffab8b64ab953b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T06:41:02,644 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 973ms, sequenceid=49, compaction requested=true 2024-11-11T06:41:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-11T06:41:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-11T06:41:02,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T06:41:02,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1240 sec 2024-11-11T06:41:02,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 3.1330 sec 2024-11-11T06:41:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:02,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:02,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T06:41:02,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/3a6ca17b31fd4c33a9f85cf33970c361 is 50, key is test_row_0/A:col10/1731307262890/Put/seqid=0 2024-11-11T06:41:02,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742071_1247 (size=14341) 2024-11-11T06:41:02,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307322901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307322904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307322969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307322969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307322970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307323009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307323019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307323219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307323229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/3a6ca17b31fd4c33a9f85cf33970c361 2024-11-11T06:41:03,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/17af9d0d74c448178d7b4fd09d281d23 is 50, key is test_row_0/B:col10/1731307262890/Put/seqid=0 2024-11-11T06:41:03,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742072_1248 (size=12001) 2024-11-11T06:41:03,505 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T06:41:03,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307323529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307323539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T06:41:03,679 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:03,679 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T06:41:03,681 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:03,683 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:03,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T06:41:03,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/17af9d0d74c448178d7b4fd09d281d23 2024-11-11T06:41:03,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/e7cc5c8cb703461ca458f212fb0c773d is 50, key is test_row_0/C:col10/1731307262890/Put/seqid=0 2024-11-11T06:41:03,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742073_1249 (size=12001) 2024-11-11T06:41:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T06:41:03,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T06:41:03,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:03,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:03,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:03,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:03,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:03,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307323979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307323979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,983 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,983 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T06:41:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:03,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:03,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307323989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:03,993 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T06:41:04,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307324039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:04,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307324059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:04,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T06:41:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:04,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:04,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:04,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:04,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:04,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/e7cc5c8cb703461ca458f212fb0c773d 2024-11-11T06:41:04,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/3a6ca17b31fd4c33a9f85cf33970c361 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/3a6ca17b31fd4c33a9f85cf33970c361 2024-11-11T06:41:04,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/3a6ca17b31fd4c33a9f85cf33970c361, entries=200, sequenceid=76, filesize=14.0 K 2024-11-11T06:41:04,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/17af9d0d74c448178d7b4fd09d281d23 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/17af9d0d74c448178d7b4fd09d281d23 2024-11-11T06:41:04,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/17af9d0d74c448178d7b4fd09d281d23, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T06:41:04,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/e7cc5c8cb703461ca458f212fb0c773d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e7cc5c8cb703461ca458f212fb0c773d 2024-11-11T06:41:04,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e7cc5c8cb703461ca458f212fb0c773d, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T06:41:04,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1310ms, sequenceid=76, compaction requested=true 2024-11-11T06:41:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:04,201 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:04,201 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:04,202 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:04,202 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:04,202 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/B is initiating minor compaction (all files) 2024-11-11T06:41:04,202 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/A is initiating minor compaction (all files) 2024-11-11T06:41:04,203 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/B in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:04,203 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/ab86d38840ae4639b8605ce343793e66, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6b3a9554c7fd45d39d66621c71a8ab7b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/17af9d0d74c448178d7b4fd09d281d23] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=46.9 K 2024-11-11T06:41:04,203 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/A in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:04,203 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2739718129ea411a9893f35e8939ee40, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/63de424bfbb940c5bee59f1a158ba400, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/7f2b21d55f0e4c5dbe509add244242ec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/3a6ca17b31fd4c33a9f85cf33970c361] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=51.4 K 2024-11-11T06:41:04,203 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ab86d38840ae4639b8605ce343793e66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731307259527 2024-11-11T06:41:04,203 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2739718129ea411a9893f35e8939ee40, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731307259527 2024-11-11T06:41:04,204 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63de424bfbb940c5bee59f1a158ba400, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731307259582 2024-11-11T06:41:04,204 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b1bfd6ddb96d4c63b2820e2f4a02b9f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731307259582 2024-11-11T06:41:04,204 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2b21d55f0e4c5dbe509add244242ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731307260723 2024-11-11T06:41:04,204 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3a9554c7fd45d39d66621c71a8ab7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731307260723 2024-11-11T06:41:04,204 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a6ca17b31fd4c33a9f85cf33970c361, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731307261815 2024-11-11T06:41:04,205 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 17af9d0d74c448178d7b4fd09d281d23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731307261815 2024-11-11T06:41:04,213 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:04,214 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/8756ba76b9d743388082411feac57543 is 50, key is test_row_0/A:col10/1731307262890/Put/seqid=0 2024-11-11T06:41:04,217 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:04,217 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/41795dc0841943ea9b22ccce7fea4c67 is 50, key is test_row_0/B:col10/1731307262890/Put/seqid=0 2024-11-11T06:41:04,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742074_1250 (size=12139) 2024-11-11T06:41:04,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742075_1251 (size=12139) 2024-11-11T06:41:04,233 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/41795dc0841943ea9b22ccce7fea4c67 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/41795dc0841943ea9b22ccce7fea4c67 2024-11-11T06:41:04,238 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/B of d55fa98fb98bf91a607d5c6eb8d4ba7d into 41795dc0841943ea9b22ccce7fea4c67(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:04,238 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:04,238 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/B, priority=12, startTime=1731307264201; duration=0sec 2024-11-11T06:41:04,238 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:04,238 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:B 2024-11-11T06:41:04,238 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:04,239 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:04,239 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/C is initiating minor compaction (all files) 2024-11-11T06:41:04,239 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/C in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:04,239 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5484da26a67c4173b2ea6a54b5d1ba36, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a816080ed9f54ac78b0e91e981737d1a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/2317c056220340da89ffab8b64ab953b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e7cc5c8cb703461ca458f212fb0c773d] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=46.9 K 2024-11-11T06:41:04,240 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 5484da26a67c4173b2ea6a54b5d1ba36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731307259527 2024-11-11T06:41:04,240 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting a816080ed9f54ac78b0e91e981737d1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731307259582 2024-11-11T06:41:04,241 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2317c056220340da89ffab8b64ab953b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731307260723 2024-11-11T06:41:04,241 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting e7cc5c8cb703461ca458f212fb0c773d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731307261815 2024-11-11T06:41:04,250 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:04,250 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/8b723c338b654314a8c503c83391369a is 50, key is test_row_0/C:col10/1731307262890/Put/seqid=0 2024-11-11T06:41:04,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742076_1252 (size=12139) 2024-11-11T06:41:04,263 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/8b723c338b654314a8c503c83391369a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/8b723c338b654314a8c503c83391369a 2024-11-11T06:41:04,268 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/C of d55fa98fb98bf91a607d5c6eb8d4ba7d into 8b723c338b654314a8c503c83391369a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:04,268 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:04,268 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/C, priority=12, startTime=1731307264201; duration=0sec 2024-11-11T06:41:04,268 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:04,268 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:C 2024-11-11T06:41:04,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T06:41:04,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:04,293 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T06:41:04,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/b88d6e5b1e8f43b18dd6b890a09c8abf is 50, key is test_row_0/A:col10/1731307262903/Put/seqid=0 2024-11-11T06:41:04,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742077_1253 (size=12001) 2024-11-11T06:41:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T06:41:04,626 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/8756ba76b9d743388082411feac57543 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/8756ba76b9d743388082411feac57543 2024-11-11T06:41:04,647 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/A of d55fa98fb98bf91a607d5c6eb8d4ba7d into 8756ba76b9d743388082411feac57543(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:04,647 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:04,647 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/A, priority=12, startTime=1731307264201; duration=0sec 2024-11-11T06:41:04,647 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:04,647 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:A 2024-11-11T06:41:04,704 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/b88d6e5b1e8f43b18dd6b890a09c8abf 2024-11-11T06:41:04,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/99b20b39fc0945868ced806c85eccb95 is 50, key is test_row_0/B:col10/1731307262903/Put/seqid=0 2024-11-11T06:41:04,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742078_1254 (size=12001) 2024-11-11T06:41:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T06:41:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:05,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:05,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307325105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307325106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,115 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/99b20b39fc0945868ced806c85eccb95 2024-11-11T06:41:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/defa83820f4342cf84d0d0d51da827d8 is 50, key is test_row_0/C:col10/1731307262903/Put/seqid=0 2024-11-11T06:41:05,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742079_1255 (size=12001) 2024-11-11T06:41:05,125 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/defa83820f4342cf84d0d0d51da827d8 2024-11-11T06:41:05,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/b88d6e5b1e8f43b18dd6b890a09c8abf as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b88d6e5b1e8f43b18dd6b890a09c8abf 2024-11-11T06:41:05,134 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b88d6e5b1e8f43b18dd6b890a09c8abf, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T06:41:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/99b20b39fc0945868ced806c85eccb95 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/99b20b39fc0945868ced806c85eccb95 2024-11-11T06:41:05,138 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/99b20b39fc0945868ced806c85eccb95, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T06:41:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/defa83820f4342cf84d0d0d51da827d8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/defa83820f4342cf84d0d0d51da827d8 2024-11-11T06:41:05,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/defa83820f4342cf84d0d0d51da827d8, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T06:41:05,143 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 850ms, sequenceid=87, compaction requested=false 2024-11-11T06:41:05,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:05,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:05,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T06:41:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T06:41:05,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T06:41:05,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4610 sec 2024-11-11T06:41:05,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4680 sec 2024-11-11T06:41:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:05,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:05,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-11T06:41:05,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/99aa456a35ae4f2cb7a6bd8b9955e987 is 50, key is test_row_0/A:col10/1731307265097/Put/seqid=0 2024-11-11T06:41:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307325229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742080_1256 (size=14341) 2024-11-11T06:41:05,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307325230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/99aa456a35ae4f2cb7a6bd8b9955e987 2024-11-11T06:41:05,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/2ff1c6cff57345f6afcbfc17cc92a9ba is 50, key is test_row_0/B:col10/1731307265097/Put/seqid=0 2024-11-11T06:41:05,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742081_1257 (size=12001) 2024-11-11T06:41:05,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/2ff1c6cff57345f6afcbfc17cc92a9ba 2024-11-11T06:41:05,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0f84a849b2ef4f0198346f69d682fbe4 is 50, key is test_row_0/C:col10/1731307265097/Put/seqid=0 2024-11-11T06:41:05,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742082_1258 (size=12001) 2024-11-11T06:41:05,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0f84a849b2ef4f0198346f69d682fbe4 2024-11-11T06:41:05,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/99aa456a35ae4f2cb7a6bd8b9955e987 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/99aa456a35ae4f2cb7a6bd8b9955e987 2024-11-11T06:41:05,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/99aa456a35ae4f2cb7a6bd8b9955e987, entries=200, sequenceid=117, filesize=14.0 K 2024-11-11T06:41:05,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/2ff1c6cff57345f6afcbfc17cc92a9ba as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2ff1c6cff57345f6afcbfc17cc92a9ba 2024-11-11T06:41:05,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2ff1c6cff57345f6afcbfc17cc92a9ba, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T06:41:05,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0f84a849b2ef4f0198346f69d682fbe4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0f84a849b2ef4f0198346f69d682fbe4 2024-11-11T06:41:05,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0f84a849b2ef4f0198346f69d682fbe4, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T06:41:05,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 54ms, sequenceid=117, compaction requested=true 2024-11-11T06:41:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:05,275 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:05,275 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:05,276 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:05,276 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/B is initiating minor compaction (all files) 2024-11-11T06:41:05,276 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/B in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:05,276 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/41795dc0841943ea9b22ccce7fea4c67, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/99b20b39fc0945868ced806c85eccb95, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2ff1c6cff57345f6afcbfc17cc92a9ba] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=35.3 K 2024-11-11T06:41:05,277 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:05,277 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/A is initiating minor compaction (all files) 2024-11-11T06:41:05,277 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/A in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:05,277 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 41795dc0841943ea9b22ccce7fea4c67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731307261815 2024-11-11T06:41:05,277 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/8756ba76b9d743388082411feac57543, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b88d6e5b1e8f43b18dd6b890a09c8abf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/99aa456a35ae4f2cb7a6bd8b9955e987] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=37.6 K 2024-11-11T06:41:05,278 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b20b39fc0945868ced806c85eccb95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731307262894 2024-11-11T06:41:05,278 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff1c6cff57345f6afcbfc17cc92a9ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731307265097 2024-11-11T06:41:05,278 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8756ba76b9d743388082411feac57543, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731307261815 2024-11-11T06:41:05,278 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting b88d6e5b1e8f43b18dd6b890a09c8abf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731307262894 2024-11-11T06:41:05,279 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99aa456a35ae4f2cb7a6bd8b9955e987, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731307265097 2024-11-11T06:41:05,286 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:05,286 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/6fd0f6f559f94632a6fff1e6a80c961e is 50, key is test_row_0/B:col10/1731307265097/Put/seqid=0 2024-11-11T06:41:05,288 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:05,288 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/f22790912cd04290ae8d7ae1a19452c5 is 50, key is test_row_0/A:col10/1731307265097/Put/seqid=0 2024-11-11T06:41:05,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742084_1260 (size=12241) 2024-11-11T06:41:05,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742083_1259 (size=12241) 2024-11-11T06:41:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:05,352 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:05,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:41:05,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/9fc0de40bfcb4770b99f779c977b4d67 is 50, key is test_row_0/A:col10/1731307265351/Put/seqid=0 2024-11-11T06:41:05,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742085_1261 (size=16681) 2024-11-11T06:41:05,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/9fc0de40bfcb4770b99f779c977b4d67 2024-11-11T06:41:05,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/2b29c1ab53634f0dae62372dcdbd08bd is 50, key is test_row_0/B:col10/1731307265351/Put/seqid=0 2024-11-11T06:41:05,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742086_1262 (size=12001) 2024-11-11T06:41:05,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307325410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307325411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307325519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307325519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,710 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/f22790912cd04290ae8d7ae1a19452c5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/f22790912cd04290ae8d7ae1a19452c5 2024-11-11T06:41:05,715 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/6fd0f6f559f94632a6fff1e6a80c961e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6fd0f6f559f94632a6fff1e6a80c961e 2024-11-11T06:41:05,715 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/A of d55fa98fb98bf91a607d5c6eb8d4ba7d into f22790912cd04290ae8d7ae1a19452c5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:05,715 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:05,715 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/A, priority=13, startTime=1731307265275; duration=0sec 2024-11-11T06:41:05,716 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:05,716 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:A 2024-11-11T06:41:05,716 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:05,717 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:05,717 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/C is initiating minor compaction (all files) 2024-11-11T06:41:05,717 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/C in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:05,717 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/8b723c338b654314a8c503c83391369a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/defa83820f4342cf84d0d0d51da827d8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0f84a849b2ef4f0198346f69d682fbe4] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=35.3 K 2024-11-11T06:41:05,717 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b723c338b654314a8c503c83391369a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731307261815 2024-11-11T06:41:05,717 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting defa83820f4342cf84d0d0d51da827d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731307262894 2024-11-11T06:41:05,718 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f84a849b2ef4f0198346f69d682fbe4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731307265097 2024-11-11T06:41:05,720 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/B of d55fa98fb98bf91a607d5c6eb8d4ba7d into 6fd0f6f559f94632a6fff1e6a80c961e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:05,720 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:05,720 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/B, priority=13, startTime=1731307265275; duration=0sec 2024-11-11T06:41:05,720 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:05,720 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:B 2024-11-11T06:41:05,723 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:05,723 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/febc77c0ca1a44cdb54a4ce88481e7f3 is 50, key is test_row_0/C:col10/1731307265097/Put/seqid=0 2024-11-11T06:41:05,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742087_1263 (size=12241) 2024-11-11T06:41:05,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307325729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307325729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:05,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/2b29c1ab53634f0dae62372dcdbd08bd 2024-11-11T06:41:05,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/e02272632afa4f848296487b903504a5 is 50, key is test_row_0/C:col10/1731307265351/Put/seqid=0 2024-11-11T06:41:05,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742088_1264 (size=12001) 2024-11-11T06:41:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T06:41:05,840 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:05,840 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T06:41:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T06:41:05,841 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:05,842 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:05,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T06:41:05,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T06:41:05,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:05,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:05,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:05,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:05,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:06,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307325999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307325999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,001 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,001 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307326009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,011 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307326039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307326039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,131 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/febc77c0ca1a44cdb54a4ce88481e7f3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/febc77c0ca1a44cdb54a4ce88481e7f3 2024-11-11T06:41:06,135 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/C of d55fa98fb98bf91a607d5c6eb8d4ba7d into febc77c0ca1a44cdb54a4ce88481e7f3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:06,135 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:06,135 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/C, priority=13, startTime=1731307265275; duration=0sec 2024-11-11T06:41:06,135 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:06,135 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:C 2024-11-11T06:41:06,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T06:41:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:06,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T06:41:06,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/e02272632afa4f848296487b903504a5 2024-11-11T06:41:06,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/9fc0de40bfcb4770b99f779c977b4d67 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9fc0de40bfcb4770b99f779c977b4d67 2024-11-11T06:41:06,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9fc0de40bfcb4770b99f779c977b4d67, entries=250, sequenceid=128, filesize=16.3 K 2024-11-11T06:41:06,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/2b29c1ab53634f0dae62372dcdbd08bd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2b29c1ab53634f0dae62372dcdbd08bd 2024-11-11T06:41:06,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2b29c1ab53634f0dae62372dcdbd08bd, entries=150, sequenceid=128, filesize=11.7 K 2024-11-11T06:41:06,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/e02272632afa4f848296487b903504a5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e02272632afa4f848296487b903504a5 2024-11-11T06:41:06,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e02272632afa4f848296487b903504a5, entries=150, sequenceid=128, filesize=11.7 K 2024-11-11T06:41:06,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 852ms, sequenceid=128, compaction requested=false 2024-11-11T06:41:06,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:06,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T06:41:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:06,300 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:41:06,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/31b11bb961fe464e80090d2f445c23ed is 50, key is test_row_0/A:col10/1731307265405/Put/seqid=0 2024-11-11T06:41:06,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742089_1265 (size=12151) 2024-11-11T06:41:06,314 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/31b11bb961fe464e80090d2f445c23ed 2024-11-11T06:41:06,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/d7e9976a1c914f588d2391f547dd1a85 is 50, key is test_row_0/B:col10/1731307265405/Put/seqid=0 2024-11-11T06:41:06,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742090_1266 (size=12151) 2024-11-11T06:41:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T06:41:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307326562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307326563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307326669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307326669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,726 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/d7e9976a1c914f588d2391f547dd1a85 2024-11-11T06:41:06,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0bc20ca6d28c4d02b754a7ada047e472 is 50, key is test_row_0/C:col10/1731307265405/Put/seqid=0 2024-11-11T06:41:06,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742091_1267 (size=12151) 2024-11-11T06:41:06,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307326879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307326879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T06:41:07,137 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0bc20ca6d28c4d02b754a7ada047e472 2024-11-11T06:41:07,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/31b11bb961fe464e80090d2f445c23ed as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/31b11bb961fe464e80090d2f445c23ed 2024-11-11T06:41:07,146 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/31b11bb961fe464e80090d2f445c23ed, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T06:41:07,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/d7e9976a1c914f588d2391f547dd1a85 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/d7e9976a1c914f588d2391f547dd1a85 2024-11-11T06:41:07,150 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/d7e9976a1c914f588d2391f547dd1a85, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T06:41:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0bc20ca6d28c4d02b754a7ada047e472 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0bc20ca6d28c4d02b754a7ada047e472 2024-11-11T06:41:07,155 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0bc20ca6d28c4d02b754a7ada047e472, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T06:41:07,156 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 855ms, sequenceid=156, compaction requested=true 2024-11-11T06:41:07,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:07,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:07,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T06:41:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T06:41:07,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T06:41:07,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3150 sec 2024-11-11T06:41:07,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.3190 sec 2024-11-11T06:41:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:07,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:07,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:41:07,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/bb81d4ac0f9b402289811283ee9b2087 is 50, key is test_row_0/A:col10/1731307266561/Put/seqid=0 2024-11-11T06:41:07,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742092_1268 (size=14541) 2024-11-11T06:41:07,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:07,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307327238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307327239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:07,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307327348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:07,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307327349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:07,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307327559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:07,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307327559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:07,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/bb81d4ac0f9b402289811283ee9b2087 2024-11-11T06:41:07,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4cc289a251da4dd990f69b93847a566f is 50, key is test_row_0/B:col10/1731307266561/Put/seqid=0 2024-11-11T06:41:07,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742093_1269 (size=12151) 2024-11-11T06:41:07,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307327869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:07,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307327869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:07,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T06:41:07,989 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:07,989 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T06:41:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T06:41:07,991 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:07,991 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:07,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:08,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4cc289a251da4dd990f69b93847a566f 2024-11-11T06:41:08,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/a3be913495ae4cefa66e6e2e8b5c6a0c is 50, key is test_row_0/C:col10/1731307266561/Put/seqid=0 2024-11-11T06:41:08,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742094_1270 (size=12151) 2024-11-11T06:41:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T06:41:08,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T06:41:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:08,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T06:41:08,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:08,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:08,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T06:41:08,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307328378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:08,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307328379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:08,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:08,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:08,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:08,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/a3be913495ae4cefa66e6e2e8b5c6a0c 2024-11-11T06:41:08,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/bb81d4ac0f9b402289811283ee9b2087 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/bb81d4ac0f9b402289811283ee9b2087 2024-11-11T06:41:08,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/bb81d4ac0f9b402289811283ee9b2087, entries=200, sequenceid=169, filesize=14.2 K 2024-11-11T06:41:08,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4cc289a251da4dd990f69b93847a566f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4cc289a251da4dd990f69b93847a566f 2024-11-11T06:41:08,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4cc289a251da4dd990f69b93847a566f, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T06:41:08,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/a3be913495ae4cefa66e6e2e8b5c6a0c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a3be913495ae4cefa66e6e2e8b5c6a0c 2024-11-11T06:41:08,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a3be913495ae4cefa66e6e2e8b5c6a0c, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T06:41:08,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1250ms, sequenceid=169, compaction requested=true 2024-11-11T06:41:08,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:08,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:08,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:08,443 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:08,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:08,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:08,443 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:08,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:08,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:08,444 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:08,444 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:08,444 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/B is initiating minor compaction (all files) 2024-11-11T06:41:08,444 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/A is initiating minor compaction (all files) 2024-11-11T06:41:08,444 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/B in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,444 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/A in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,444 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6fd0f6f559f94632a6fff1e6a80c961e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2b29c1ab53634f0dae62372dcdbd08bd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/d7e9976a1c914f588d2391f547dd1a85, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4cc289a251da4dd990f69b93847a566f] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=47.4 K 2024-11-11T06:41:08,444 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/f22790912cd04290ae8d7ae1a19452c5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9fc0de40bfcb4770b99f779c977b4d67, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/31b11bb961fe464e80090d2f445c23ed, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/bb81d4ac0f9b402289811283ee9b2087] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=54.3 K 2024-11-11T06:41:08,444 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fd0f6f559f94632a6fff1e6a80c961e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731307265097 2024-11-11T06:41:08,444 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22790912cd04290ae8d7ae1a19452c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731307265097 2024-11-11T06:41:08,445 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc0de40bfcb4770b99f779c977b4d67, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731307265229 2024-11-11T06:41:08,445 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b29c1ab53634f0dae62372dcdbd08bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731307265341 2024-11-11T06:41:08,445 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31b11bb961fe464e80090d2f445c23ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731307265405 2024-11-11T06:41:08,445 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e9976a1c914f588d2391f547dd1a85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731307265405 2024-11-11T06:41:08,445 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc289a251da4dd990f69b93847a566f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731307266561 2024-11-11T06:41:08,445 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb81d4ac0f9b402289811283ee9b2087, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731307266552 2024-11-11T06:41:08,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T06:41:08,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,449 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T06:41:08,454 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:08,454 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/cfc3c58fbe4a42ac8b2fb2cc1e7e6336 is 50, key is test_row_0/A:col10/1731307266561/Put/seqid=0 2024-11-11T06:41:08,455 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:08,455 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/461072d74bf54c8fb5e21ab11ee434e8 is 50, key is test_row_0/B:col10/1731307266561/Put/seqid=0 2024-11-11T06:41:08,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/e06c7a15e9b3443e809ac04d33448210 is 50, key is test_row_0/A:col10/1731307267238/Put/seqid=0 2024-11-11T06:41:08,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742095_1271 (size=12527) 2024-11-11T06:41:08,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742096_1272 (size=12527) 2024-11-11T06:41:08,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742097_1273 (size=12151) 2024-11-11T06:41:08,471 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/e06c7a15e9b3443e809ac04d33448210 2024-11-11T06:41:08,477 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/461072d74bf54c8fb5e21ab11ee434e8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/461072d74bf54c8fb5e21ab11ee434e8 2024-11-11T06:41:08,478 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/cfc3c58fbe4a42ac8b2fb2cc1e7e6336 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/cfc3c58fbe4a42ac8b2fb2cc1e7e6336 2024-11-11T06:41:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/0543b39fa7d94b75ba7b335058ac447d is 50, key is test_row_0/B:col10/1731307267238/Put/seqid=0 2024-11-11T06:41:08,486 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/B of d55fa98fb98bf91a607d5c6eb8d4ba7d into 461072d74bf54c8fb5e21ab11ee434e8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:08,486 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:08,486 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/A of d55fa98fb98bf91a607d5c6eb8d4ba7d into cfc3c58fbe4a42ac8b2fb2cc1e7e6336(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:08,486 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/B, priority=12, startTime=1731307268443; duration=0sec 2024-11-11T06:41:08,486 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:08,486 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/A, priority=12, startTime=1731307268443; duration=0sec 2024-11-11T06:41:08,486 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:08,486 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:B 2024-11-11T06:41:08,486 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:08,486 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:A 2024-11-11T06:41:08,486 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:08,488 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:08,488 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/C is initiating minor compaction (all files) 2024-11-11T06:41:08,488 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/C in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,488 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/febc77c0ca1a44cdb54a4ce88481e7f3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e02272632afa4f848296487b903504a5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0bc20ca6d28c4d02b754a7ada047e472, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a3be913495ae4cefa66e6e2e8b5c6a0c] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=47.4 K 2024-11-11T06:41:08,489 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting febc77c0ca1a44cdb54a4ce88481e7f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731307265097 2024-11-11T06:41:08,489 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting e02272632afa4f848296487b903504a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731307265341 2024-11-11T06:41:08,489 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc20ca6d28c4d02b754a7ada047e472, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731307265405 2024-11-11T06:41:08,490 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting a3be913495ae4cefa66e6e2e8b5c6a0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731307266561 2024-11-11T06:41:08,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742098_1274 (size=12151) 2024-11-11T06:41:08,493 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/0543b39fa7d94b75ba7b335058ac447d 2024-11-11T06:41:08,500 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#C#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:08,500 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/4d83d3e2d81f498ea3c3b350b05569cd is 50, key is test_row_0/C:col10/1731307266561/Put/seqid=0 2024-11-11T06:41:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/f6de3281198d4d298154daae25a66801 is 50, key is test_row_0/C:col10/1731307267238/Put/seqid=0 2024-11-11T06:41:08,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742099_1275 (size=12527) 2024-11-11T06:41:08,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742100_1276 (size=12151) 2024-11-11T06:41:08,513 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/f6de3281198d4d298154daae25a66801 2024-11-11T06:41:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/e06c7a15e9b3443e809ac04d33448210 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e06c7a15e9b3443e809ac04d33448210 2024-11-11T06:41:08,521 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e06c7a15e9b3443e809ac04d33448210, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T06:41:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/0543b39fa7d94b75ba7b335058ac447d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0543b39fa7d94b75ba7b335058ac447d 2024-11-11T06:41:08,526 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0543b39fa7d94b75ba7b335058ac447d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T06:41:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/f6de3281198d4d298154daae25a66801 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/f6de3281198d4d298154daae25a66801 2024-11-11T06:41:08,530 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/f6de3281198d4d298154daae25a66801, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T06:41:08,531 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 82ms, sequenceid=192, compaction requested=false 2024-11-11T06:41:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T06:41:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T06:41:08,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T06:41:08,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 541 msec 2024-11-11T06:41:08,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 548 msec 2024-11-11T06:41:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T06:41:08,619 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:08,619 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T06:41:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T06:41:08,621 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:08,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:08,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T06:41:08,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T06:41:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T06:41:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T06:41:08,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T06:41:08,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-11T06:41:08,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 159 msec 2024-11-11T06:41:08,914 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/4d83d3e2d81f498ea3c3b350b05569cd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4d83d3e2d81f498ea3c3b350b05569cd 2024-11-11T06:41:08,918 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/C of d55fa98fb98bf91a607d5c6eb8d4ba7d into 4d83d3e2d81f498ea3c3b350b05569cd(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:08,918 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:08,918 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/C, priority=12, startTime=1731307268443; duration=0sec 2024-11-11T06:41:08,918 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:08,918 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:C 2024-11-11T06:41:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T06:41:08,939 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:08,940 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T06:41:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T06:41:08,942 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:08,943 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:08,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T06:41:09,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T06:41:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T06:41:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T06:41:09,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T06:41:09,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-11T06:41:09,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 160 msec 2024-11-11T06:41:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T06:41:09,259 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:09,260 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T06:41:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T06:41:09,261 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:09,262 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:09,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T06:41:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:09,401 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:09,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:41:09,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/ee399e574f0b4966b1cafb37f7ece10a is 50, key is test_row_0/A:col10/1731307269394/Put/seqid=0 2024-11-11T06:41:09,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T06:41:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742101_1277 (size=14541) 2024-11-11T06:41:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307329446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:09,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307329447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:09,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307329559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307329559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:09,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T06:41:09,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:09,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T06:41:09,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T06:41:09,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:09,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:09,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307329769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:09,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307329769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:09,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/ee399e574f0b4966b1cafb37f7ece10a 2024-11-11T06:41:09,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/3f73f3b935104f8d86d3f57e660a16ab is 50, key is test_row_0/B:col10/1731307269394/Put/seqid=0 2024-11-11T06:41:09,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742102_1278 (size=12151) 2024-11-11T06:41:09,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/3f73f3b935104f8d86d3f57e660a16ab 2024-11-11T06:41:09,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/74d763a6c04b4ac1affb56b02dc01292 is 50, key is test_row_0/C:col10/1731307269394/Put/seqid=0 2024-11-11T06:41:09,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742103_1279 (size=12151) 2024-11-11T06:41:09,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/74d763a6c04b4ac1affb56b02dc01292 2024-11-11T06:41:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/ee399e574f0b4966b1cafb37f7ece10a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/ee399e574f0b4966b1cafb37f7ece10a 2024-11-11T06:41:09,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/ee399e574f0b4966b1cafb37f7ece10a, entries=200, sequenceid=208, filesize=14.2 K 2024-11-11T06:41:09,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/3f73f3b935104f8d86d3f57e660a16ab as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/3f73f3b935104f8d86d3f57e660a16ab 2024-11-11T06:41:09,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/3f73f3b935104f8d86d3f57e660a16ab, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T06:41:09,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/74d763a6c04b4ac1affb56b02dc01292 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/74d763a6c04b4ac1affb56b02dc01292 2024-11-11T06:41:09,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/74d763a6c04b4ac1affb56b02dc01292, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T06:41:09,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 460ms, sequenceid=208, compaction requested=true 2024-11-11T06:41:09,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:09,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:09,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:09,861 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:09,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:09,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:09,861 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:09,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:09,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:09,862 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:09,862 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/A is initiating minor compaction (all files) 2024-11-11T06:41:09,862 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:09,862 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/A in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,862 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/B is initiating minor compaction (all files) 2024-11-11T06:41:09,862 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/B in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,862 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/cfc3c58fbe4a42ac8b2fb2cc1e7e6336, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e06c7a15e9b3443e809ac04d33448210, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/ee399e574f0b4966b1cafb37f7ece10a] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=38.3 K 2024-11-11T06:41:09,863 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/461072d74bf54c8fb5e21ab11ee434e8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0543b39fa7d94b75ba7b335058ac447d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/3f73f3b935104f8d86d3f57e660a16ab] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=36.0 K 2024-11-11T06:41:09,863 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc3c58fbe4a42ac8b2fb2cc1e7e6336, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731307266561 2024-11-11T06:41:09,864 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 461072d74bf54c8fb5e21ab11ee434e8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731307266561 2024-11-11T06:41:09,864 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06c7a15e9b3443e809ac04d33448210, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731307267231 2024-11-11T06:41:09,864 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0543b39fa7d94b75ba7b335058ac447d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731307267231 2024-11-11T06:41:09,864 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee399e574f0b4966b1cafb37f7ece10a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307269394 2024-11-11T06:41:09,864 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f73f3b935104f8d86d3f57e660a16ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307269394 2024-11-11T06:41:09,872 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:09,872 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:09,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T06:41:09,872 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/92f514151fd5409ab58ca0d7a926bd44 is 50, key is test_row_0/B:col10/1731307269394/Put/seqid=0 2024-11-11T06:41:09,872 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/b648d02189f14acdbe2652d56c71470e is 50, key is test_row_0/A:col10/1731307269394/Put/seqid=0 2024-11-11T06:41:09,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:09,872 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:41:09,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742104_1280 (size=12629) 2024-11-11T06:41:09,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/0998722ca1394a97bf041a6836bc957e is 50, key is test_row_0/A:col10/1731307269446/Put/seqid=0 2024-11-11T06:41:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T06:41:09,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742105_1281 (size=12629) 2024-11-11T06:41:09,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742106_1282 (size=12151) 2024-11-11T06:41:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:10,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:10,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307330044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307330044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307330047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307330078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307330079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307330159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307330159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307330160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,284 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/92f514151fd5409ab58ca0d7a926bd44 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/92f514151fd5409ab58ca0d7a926bd44 2024-11-11T06:41:10,289 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/B of d55fa98fb98bf91a607d5c6eb8d4ba7d into 92f514151fd5409ab58ca0d7a926bd44(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:10,289 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:10,289 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/B, priority=13, startTime=1731307269861; duration=0sec 2024-11-11T06:41:10,289 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:10,289 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:B 2024-11-11T06:41:10,289 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:10,290 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:10,290 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/C is initiating minor compaction (all files) 2024-11-11T06:41:10,290 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/C in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:10,291 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4d83d3e2d81f498ea3c3b350b05569cd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/f6de3281198d4d298154daae25a66801, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/74d763a6c04b4ac1affb56b02dc01292] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=36.0 K 2024-11-11T06:41:10,291 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d83d3e2d81f498ea3c3b350b05569cd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731307266561 2024-11-11T06:41:10,291 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting f6de3281198d4d298154daae25a66801, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731307267231 2024-11-11T06:41:10,292 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d763a6c04b4ac1affb56b02dc01292, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307269394 2024-11-11T06:41:10,297 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:10,298 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/48cc7df465094d6abf48c4cfd667fc8f is 50, key is test_row_0/C:col10/1731307269394/Put/seqid=0 2024-11-11T06:41:10,304 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/b648d02189f14acdbe2652d56c71470e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b648d02189f14acdbe2652d56c71470e 2024-11-11T06:41:10,308 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/A of d55fa98fb98bf91a607d5c6eb8d4ba7d into b648d02189f14acdbe2652d56c71470e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:10,308 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:10,308 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/A, priority=13, startTime=1731307269861; duration=0sec 2024-11-11T06:41:10,309 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:10,309 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:A 2024-11-11T06:41:10,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742107_1283 (size=12629) 2024-11-11T06:41:10,320 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/0998722ca1394a97bf041a6836bc957e 2024-11-11T06:41:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/bfc540b99b744482a51a89dbf9ed2975 is 50, key is test_row_0/B:col10/1731307269446/Put/seqid=0 2024-11-11T06:41:10,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742108_1284 (size=12151) 2024-11-11T06:41:10,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307330369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307330369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307330369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T06:41:10,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307330589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307330589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307330679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307330679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307330679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:10,716 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/48cc7df465094d6abf48c4cfd667fc8f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/48cc7df465094d6abf48c4cfd667fc8f 2024-11-11T06:41:10,721 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/C of d55fa98fb98bf91a607d5c6eb8d4ba7d into 48cc7df465094d6abf48c4cfd667fc8f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:10,721 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:10,721 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/C, priority=13, startTime=1731307269861; duration=0sec 2024-11-11T06:41:10,721 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:10,721 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:C 2024-11-11T06:41:10,742 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/bfc540b99b744482a51a89dbf9ed2975 2024-11-11T06:41:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/4a215fc814b54d0ba381d3eb4c507d4d is 50, key is test_row_0/C:col10/1731307269446/Put/seqid=0 2024-11-11T06:41:10,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742109_1285 (size=12151) 2024-11-11T06:41:11,154 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/4a215fc814b54d0ba381d3eb4c507d4d 2024-11-11T06:41:11,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/0998722ca1394a97bf041a6836bc957e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/0998722ca1394a97bf041a6836bc957e 2024-11-11T06:41:11,162 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/0998722ca1394a97bf041a6836bc957e, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T06:41:11,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/bfc540b99b744482a51a89dbf9ed2975 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/bfc540b99b744482a51a89dbf9ed2975 2024-11-11T06:41:11,167 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/bfc540b99b744482a51a89dbf9ed2975, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T06:41:11,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/4a215fc814b54d0ba381d3eb4c507d4d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4a215fc814b54d0ba381d3eb4c507d4d 2024-11-11T06:41:11,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4a215fc814b54d0ba381d3eb4c507d4d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T06:41:11,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1300ms, sequenceid=233, compaction requested=false 2024-11-11T06:41:11,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:11,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:11,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T06:41:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T06:41:11,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T06:41:11,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9120 sec 2024-11-11T06:41:11,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.9160 sec 2024-11-11T06:41:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:11,191 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:11,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:41:11,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/2d1b80cde1924badb02b87cd53c76230 is 50, key is test_row_0/A:col10/1731307271190/Put/seqid=0 2024-11-11T06:41:11,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742110_1286 (size=16931) 2024-11-11T06:41:11,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307331221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307331222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307331223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307331329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307331329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307331329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T06:41:11,409 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:11,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T06:41:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T06:41:11,411 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:11,411 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:11,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T06:41:11,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307331539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307331539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307331539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T06:41:11,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:11,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:11,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:11,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307331599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,601 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/2d1b80cde1924badb02b87cd53c76230 2024-11-11T06:41:11,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/57cbcdab351e49689d9d42c98a138e18 is 50, key is test_row_0/B:col10/1731307271190/Put/seqid=0 2024-11-11T06:41:11,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307331609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,612 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742111_1287 (size=12151) 2024-11-11T06:41:11,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T06:41:11,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:11,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:11,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:11,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T06:41:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307331849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307331849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307331849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:11,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T06:41:11,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:11,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:11,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:11,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:11,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T06:41:12,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/57cbcdab351e49689d9d42c98a138e18 2024-11-11T06:41:12,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/94308e43657a4ba3b1298c8fa9862e6a is 50, key is test_row_0/C:col10/1731307271190/Put/seqid=0 2024-11-11T06:41:12,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T06:41:12,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:12,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742112_1288 (size=12151) 2024-11-11T06:41:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T06:41:12,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T06:41:12,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:12,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T06:41:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:12,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:12,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307332359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:12,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307332359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:12,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307332359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:12,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/94308e43657a4ba3b1298c8fa9862e6a 2024-11-11T06:41:12,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/2d1b80cde1924badb02b87cd53c76230 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2d1b80cde1924badb02b87cd53c76230 2024-11-11T06:41:12,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2d1b80cde1924badb02b87cd53c76230, entries=250, sequenceid=250, filesize=16.5 K 2024-11-11T06:41:12,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/57cbcdab351e49689d9d42c98a138e18 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/57cbcdab351e49689d9d42c98a138e18 2024-11-11T06:41:12,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/57cbcdab351e49689d9d42c98a138e18, entries=150, sequenceid=250, filesize=11.9 K 2024-11-11T06:41:12,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/94308e43657a4ba3b1298c8fa9862e6a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/94308e43657a4ba3b1298c8fa9862e6a 2024-11-11T06:41:12,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/94308e43657a4ba3b1298c8fa9862e6a, entries=150, sequenceid=250, filesize=11.9 K 2024-11-11T06:41:12,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1250ms, sequenceid=250, compaction requested=true 2024-11-11T06:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:12,441 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:12,441 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:12,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:12,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:12,442 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:12,443 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/A is initiating minor compaction (all files) 2024-11-11T06:41:12,443 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/A in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,443 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b648d02189f14acdbe2652d56c71470e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/0998722ca1394a97bf041a6836bc957e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2d1b80cde1924badb02b87cd53c76230] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=40.7 K 2024-11-11T06:41:12,443 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:12,443 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/B is initiating minor compaction (all files) 2024-11-11T06:41:12,443 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/B in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,443 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/92f514151fd5409ab58ca0d7a926bd44, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/bfc540b99b744482a51a89dbf9ed2975, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/57cbcdab351e49689d9d42c98a138e18] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=36.1 K 2024-11-11T06:41:12,443 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting b648d02189f14acdbe2652d56c71470e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307269394 2024-11-11T06:41:12,443 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f514151fd5409ab58ca0d7a926bd44, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307269394 2024-11-11T06:41:12,444 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0998722ca1394a97bf041a6836bc957e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731307269435 2024-11-11T06:41:12,444 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc540b99b744482a51a89dbf9ed2975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731307269435 2024-11-11T06:41:12,444 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 57cbcdab351e49689d9d42c98a138e18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731307271190 2024-11-11T06:41:12,444 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d1b80cde1924badb02b87cd53c76230, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731307270041 2024-11-11T06:41:12,452 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:12,453 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/454373779d354494a225428b4504322d is 50, key is test_row_0/A:col10/1731307271190/Put/seqid=0 2024-11-11T06:41:12,455 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:12,456 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/b21d9f8a67204dbaa7eeff3e9a90aad2 is 50, key is test_row_0/B:col10/1731307271190/Put/seqid=0 2024-11-11T06:41:12,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742114_1290 (size=12731) 2024-11-11T06:41:12,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742113_1289 (size=12731) 2024-11-11T06:41:12,470 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/454373779d354494a225428b4504322d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/454373779d354494a225428b4504322d 2024-11-11T06:41:12,475 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/A of d55fa98fb98bf91a607d5c6eb8d4ba7d into 454373779d354494a225428b4504322d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:12,475 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:12,475 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/A, priority=13, startTime=1731307272441; duration=0sec 2024-11-11T06:41:12,475 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:12,475 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:A 2024-11-11T06:41:12,476 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:12,477 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:12,477 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/C is initiating minor compaction (all files) 2024-11-11T06:41:12,477 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/C in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,477 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/48cc7df465094d6abf48c4cfd667fc8f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4a215fc814b54d0ba381d3eb4c507d4d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/94308e43657a4ba3b1298c8fa9862e6a] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=36.1 K 2024-11-11T06:41:12,478 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48cc7df465094d6abf48c4cfd667fc8f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731307269394 2024-11-11T06:41:12,478 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a215fc814b54d0ba381d3eb4c507d4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731307269435 2024-11-11T06:41:12,478 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94308e43657a4ba3b1298c8fa9862e6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731307271190 2024-11-11T06:41:12,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T06:41:12,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:12,480 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T06:41:12,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/173aa155bccc44ef889063023781bc2a is 50, key is test_row_0/A:col10/1731307271217/Put/seqid=0 2024-11-11T06:41:12,487 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:12,487 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/3a4f1223835542048a51a31c64dea719 is 50, key is test_row_0/C:col10/1731307271190/Put/seqid=0 2024-11-11T06:41:12,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742115_1291 (size=12301) 2024-11-11T06:41:12,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742116_1292 (size=12731) 2024-11-11T06:41:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T06:41:12,871 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/b21d9f8a67204dbaa7eeff3e9a90aad2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b21d9f8a67204dbaa7eeff3e9a90aad2 2024-11-11T06:41:12,878 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/B of d55fa98fb98bf91a607d5c6eb8d4ba7d into b21d9f8a67204dbaa7eeff3e9a90aad2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:12,878 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:12,878 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/B, priority=13, startTime=1731307272441; duration=0sec 2024-11-11T06:41:12,878 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:12,878 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:B 2024-11-11T06:41:12,896 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/173aa155bccc44ef889063023781bc2a 2024-11-11T06:41:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/c07e2491ea7646c3881598b35de847f6 is 50, key is test_row_0/B:col10/1731307271217/Put/seqid=0 2024-11-11T06:41:12,913 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/3a4f1223835542048a51a31c64dea719 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/3a4f1223835542048a51a31c64dea719 2024-11-11T06:41:12,921 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/C of d55fa98fb98bf91a607d5c6eb8d4ba7d into 3a4f1223835542048a51a31c64dea719(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:12,921 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:12,921 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/C, priority=13, startTime=1731307272441; duration=0sec 2024-11-11T06:41:12,921 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:12,921 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:C 2024-11-11T06:41:12,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742117_1293 (size=12301) 2024-11-11T06:41:13,324 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/c07e2491ea7646c3881598b35de847f6 2024-11-11T06:41:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/05677989c4144179a1f26c61f25b4890 is 50, key is test_row_0/C:col10/1731307271217/Put/seqid=0 2024-11-11T06:41:13,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742118_1294 (size=12301) 2024-11-11T06:41:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:13,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307333390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307333391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307333394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307333499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307333499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307333499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T06:41:13,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307333609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,613 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307333630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,633 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307333709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307333710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307333710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:13,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:13,755 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/05677989c4144179a1f26c61f25b4890 2024-11-11T06:41:13,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/173aa155bccc44ef889063023781bc2a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/173aa155bccc44ef889063023781bc2a 2024-11-11T06:41:13,767 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/173aa155bccc44ef889063023781bc2a, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T06:41:13,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/c07e2491ea7646c3881598b35de847f6 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/c07e2491ea7646c3881598b35de847f6 2024-11-11T06:41:13,772 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/c07e2491ea7646c3881598b35de847f6, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T06:41:13,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/05677989c4144179a1f26c61f25b4890 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/05677989c4144179a1f26c61f25b4890 2024-11-11T06:41:13,779 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/05677989c4144179a1f26c61f25b4890, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T06:41:13,780 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1300ms, sequenceid=273, compaction requested=false 2024-11-11T06:41:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T06:41:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T06:41:13,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T06:41:13,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3710 sec 2024-11-11T06:41:13,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.3750 sec 2024-11-11T06:41:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:14,021 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:14,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T06:41:14,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/67f02882f9174a8a92285c95b193a1c0 is 50, key is test_row_0/A:col10/1731307274020/Put/seqid=0 2024-11-11T06:41:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742119_1295 (size=14741) 2024-11-11T06:41:14,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/67f02882f9174a8a92285c95b193a1c0 2024-11-11T06:41:14,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/852f1efd9c70427b9596d3c6c600a1d3 is 50, key is test_row_0/B:col10/1731307274020/Put/seqid=0 2024-11-11T06:41:14,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742120_1296 (size=12301) 2024-11-11T06:41:14,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307334067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307334067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307334068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307334179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307334179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307334180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307334389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307334389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307334389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/852f1efd9c70427b9596d3c6c600a1d3 2024-11-11T06:41:14,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/7d7cf471f8124bbaa70e824918308c52 is 50, key is test_row_0/C:col10/1731307274020/Put/seqid=0 2024-11-11T06:41:14,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742121_1297 (size=12301) 2024-11-11T06:41:14,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307334699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307334699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307334699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:14,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/7d7cf471f8124bbaa70e824918308c52 2024-11-11T06:41:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/67f02882f9174a8a92285c95b193a1c0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/67f02882f9174a8a92285c95b193a1c0 2024-11-11T06:41:14,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/67f02882f9174a8a92285c95b193a1c0, entries=200, sequenceid=290, filesize=14.4 K 2024-11-11T06:41:14,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/852f1efd9c70427b9596d3c6c600a1d3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/852f1efd9c70427b9596d3c6c600a1d3 2024-11-11T06:41:14,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/852f1efd9c70427b9596d3c6c600a1d3, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T06:41:14,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/7d7cf471f8124bbaa70e824918308c52 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/7d7cf471f8124bbaa70e824918308c52 2024-11-11T06:41:14,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/7d7cf471f8124bbaa70e824918308c52, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T06:41:14,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 900ms, sequenceid=290, compaction requested=true 2024-11-11T06:41:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:14,921 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:14,921 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:14,922 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:14,922 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:14,922 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/B is initiating minor compaction (all files) 2024-11-11T06:41:14,922 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/A is initiating minor compaction (all files) 2024-11-11T06:41:14,922 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/A in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:14,922 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/B in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:14,923 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/454373779d354494a225428b4504322d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/173aa155bccc44ef889063023781bc2a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/67f02882f9174a8a92285c95b193a1c0] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=38.8 K 2024-11-11T06:41:14,923 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b21d9f8a67204dbaa7eeff3e9a90aad2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/c07e2491ea7646c3881598b35de847f6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/852f1efd9c70427b9596d3c6c600a1d3] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=36.5 K 2024-11-11T06:41:14,923 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b21d9f8a67204dbaa7eeff3e9a90aad2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731307271190 2024-11-11T06:41:14,923 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454373779d354494a225428b4504322d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731307271190 2024-11-11T06:41:14,923 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c07e2491ea7646c3881598b35de847f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731307271217 2024-11-11T06:41:14,923 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 173aa155bccc44ef889063023781bc2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731307271217 2024-11-11T06:41:14,923 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 852f1efd9c70427b9596d3c6c600a1d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731307273389 2024-11-11T06:41:14,923 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f02882f9174a8a92285c95b193a1c0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731307273389 2024-11-11T06:41:14,929 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:14,929 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/86830409963e436ba8a1a9f86931db0b is 50, key is test_row_0/A:col10/1731307274020/Put/seqid=0 2024-11-11T06:41:14,936 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:14,936 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/0a2e52580d054104b0ab7d70fb50c2ed is 50, key is test_row_0/B:col10/1731307274020/Put/seqid=0 2024-11-11T06:41:14,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742122_1298 (size=12983) 2024-11-11T06:41:14,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742123_1299 (size=12983) 2024-11-11T06:41:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:15,211 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:15,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T06:41:15,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/e2ee49da487641668a90fb968f1065ea is 50, key is test_row_0/A:col10/1731307275212/Put/seqid=0 2024-11-11T06:41:15,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742124_1300 (size=19621) 2024-11-11T06:41:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307335227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307335227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307335228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307335339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307335339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307335339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,346 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/0a2e52580d054104b0ab7d70fb50c2ed as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0a2e52580d054104b0ab7d70fb50c2ed 2024-11-11T06:41:15,347 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/86830409963e436ba8a1a9f86931db0b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/86830409963e436ba8a1a9f86931db0b 2024-11-11T06:41:15,351 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/B of d55fa98fb98bf91a607d5c6eb8d4ba7d into 0a2e52580d054104b0ab7d70fb50c2ed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:15,351 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:15,351 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/B, priority=13, startTime=1731307274921; duration=0sec 2024-11-11T06:41:15,351 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:15,351 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:B 2024-11-11T06:41:15,351 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:15,353 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/A of d55fa98fb98bf91a607d5c6eb8d4ba7d into 86830409963e436ba8a1a9f86931db0b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:15,353 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:15,353 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/A, priority=13, startTime=1731307274921; duration=0sec 2024-11-11T06:41:15,353 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:15,353 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:15,353 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:A 2024-11-11T06:41:15,353 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/C is initiating minor compaction (all files) 2024-11-11T06:41:15,353 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/C in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:15,353 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/3a4f1223835542048a51a31c64dea719, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/05677989c4144179a1f26c61f25b4890, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/7d7cf471f8124bbaa70e824918308c52] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=36.5 K 2024-11-11T06:41:15,354 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4f1223835542048a51a31c64dea719, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731307271190 2024-11-11T06:41:15,354 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 05677989c4144179a1f26c61f25b4890, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731307271217 2024-11-11T06:41:15,354 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d7cf471f8124bbaa70e824918308c52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731307273389 2024-11-11T06:41:15,365 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:15,365 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/5e8f8bb98f8245dba1a6cc667e317444 is 50, key is test_row_0/C:col10/1731307274020/Put/seqid=0 2024-11-11T06:41:15,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742125_1301 (size=12983) 2024-11-11T06:41:15,375 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/5e8f8bb98f8245dba1a6cc667e317444 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5e8f8bb98f8245dba1a6cc667e317444 2024-11-11T06:41:15,381 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/C of d55fa98fb98bf91a607d5c6eb8d4ba7d into 5e8f8bb98f8245dba1a6cc667e317444(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:15,381 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:15,381 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/C, priority=13, startTime=1731307274921; duration=0sec 2024-11-11T06:41:15,381 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:15,381 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:C 2024-11-11T06:41:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307335549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307335549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307335549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T06:41:15,569 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:15,570 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-11T06:41:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T06:41:15,571 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:15,573 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:15,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:15,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/e2ee49da487641668a90fb968f1065ea 2024-11-11T06:41:15,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/44b0155bbbad4ff1b18fe74108e2bb7d is 50, key is test_row_0/B:col10/1731307275212/Put/seqid=0 2024-11-11T06:41:15,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742126_1302 (size=12301) 2024-11-11T06:41:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T06:41:15,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T06:41:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:15,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:15,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307335858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307335859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307335859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:15,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:15,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:15,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T06:41:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:15,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T06:41:16,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T06:41:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:16,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:16,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:16,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/44b0155bbbad4ff1b18fe74108e2bb7d 2024-11-11T06:41:16,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/01174fc685d84d7387af8ed855dcbf07 is 50, key is test_row_0/C:col10/1731307275212/Put/seqid=0 2024-11-11T06:41:16,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742127_1303 (size=12301) 2024-11-11T06:41:16,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T06:41:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:16,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T06:41:16,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T06:41:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:16,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:16,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307336368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:16,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307336369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:16,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307336369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:16,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/01174fc685d84d7387af8ed855dcbf07 2024-11-11T06:41:16,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/e2ee49da487641668a90fb968f1065ea as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e2ee49da487641668a90fb968f1065ea 2024-11-11T06:41:16,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e2ee49da487641668a90fb968f1065ea, entries=300, sequenceid=314, filesize=19.2 K 2024-11-11T06:41:16,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/44b0155bbbad4ff1b18fe74108e2bb7d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/44b0155bbbad4ff1b18fe74108e2bb7d 2024-11-11T06:41:16,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/44b0155bbbad4ff1b18fe74108e2bb7d, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T06:41:16,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/01174fc685d84d7387af8ed855dcbf07 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/01174fc685d84d7387af8ed855dcbf07 2024-11-11T06:41:16,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/01174fc685d84d7387af8ed855dcbf07, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T06:41:16,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1275ms, sequenceid=314, compaction requested=false 2024-11-11T06:41:16,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:16,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T06:41:16,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:16,489 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:41:16,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/146206a54be2467b8c7a05a2aefea181 is 50, key is test_row_0/A:col10/1731307275226/Put/seqid=0 2024-11-11T06:41:16,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742128_1304 (size=12301) 2024-11-11T06:41:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T06:41:16,909 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/146206a54be2467b8c7a05a2aefea181 2024-11-11T06:41:16,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/5f57f4213eb1480c9591f329dba77aaf is 50, key is test_row_0/B:col10/1731307275226/Put/seqid=0 2024-11-11T06:41:16,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742129_1305 (size=12301) 2024-11-11T06:41:17,319 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/5f57f4213eb1480c9591f329dba77aaf 2024-11-11T06:41:17,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/ea1fc7b4b9bf48518c10ac7370accf7c is 50, key is test_row_0/C:col10/1731307275226/Put/seqid=0 2024-11-11T06:41:17,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742130_1306 (size=12301) 2024-11-11T06:41:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:17,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. as already flushing 2024-11-11T06:41:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307337408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307337409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307337410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307337519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307337519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307337519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1731307337649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,650 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731307337669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,671 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in d55fa98fb98bf91a607d5c6eb8d4ba7d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T06:41:17,729 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/ea1fc7b4b9bf48518c10ac7370accf7c 2024-11-11T06:41:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307337729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307337729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/146206a54be2467b8c7a05a2aefea181 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/146206a54be2467b8c7a05a2aefea181 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307337730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:17,736 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/146206a54be2467b8c7a05a2aefea181, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T06:41:17,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/5f57f4213eb1480c9591f329dba77aaf as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/5f57f4213eb1480c9591f329dba77aaf 2024-11-11T06:41:17,740 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/5f57f4213eb1480c9591f329dba77aaf, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T06:41:17,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/ea1fc7b4b9bf48518c10ac7370accf7c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/ea1fc7b4b9bf48518c10ac7370accf7c 2024-11-11T06:41:17,744 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/ea1fc7b4b9bf48518c10ac7370accf7c, entries=150, sequenceid=329, filesize=12.0 K 2024-11-11T06:41:17,745 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1256ms, sequenceid=329, compaction requested=true 2024-11-11T06:41:17,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:17,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:17,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-11T06:41:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-11T06:41:17,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T06:41:17,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1730 sec 2024-11-11T06:41:17,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.1780 sec 2024-11-11T06:41:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:18,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:41:18,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/9b30cf237c2f4d1c8516645c5003d80a is 50, key is test_row_0/A:col10/1731307278040/Put/seqid=0 2024-11-11T06:41:18,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742131_1307 (size=17181) 2024-11-11T06:41:18,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307338050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307338053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307338053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307338159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307338159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307338159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307338369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307338369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307338369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/9b30cf237c2f4d1c8516645c5003d80a 2024-11-11T06:41:18,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4c0e2c91fa454b79958768b3b63406ed is 50, key is test_row_0/B:col10/1731307278040/Put/seqid=0 2024-11-11T06:41:18,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742132_1308 (size=12301) 2024-11-11T06:41:18,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307338679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307338679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307338679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:18,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:18,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4c0e2c91fa454b79958768b3b63406ed 2024-11-11T06:41:18,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/53f551f5588946ac8151465cb6b59255 is 50, key is test_row_0/C:col10/1731307278040/Put/seqid=0 2024-11-11T06:41:18,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742133_1309 (size=12301) 2024-11-11T06:41:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49510 deadline: 1731307339189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:19,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731307339189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:19,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49516 deadline: 1731307339190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:41:19,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/53f551f5588946ac8151465cb6b59255 2024-11-11T06:41:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/9b30cf237c2f4d1c8516645c5003d80a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9b30cf237c2f4d1c8516645c5003d80a 2024-11-11T06:41:19,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9b30cf237c2f4d1c8516645c5003d80a, entries=250, sequenceid=356, filesize=16.8 K 2024-11-11T06:41:19,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4c0e2c91fa454b79958768b3b63406ed as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4c0e2c91fa454b79958768b3b63406ed 2024-11-11T06:41:19,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4c0e2c91fa454b79958768b3b63406ed, entries=150, sequenceid=356, filesize=12.0 K 2024-11-11T06:41:19,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/53f551f5588946ac8151465cb6b59255 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/53f551f5588946ac8151465cb6b59255 2024-11-11T06:41:19,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/53f551f5588946ac8151465cb6b59255, entries=150, sequenceid=356, filesize=12.0 K 2024-11-11T06:41:19,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1283ms, sequenceid=356, compaction requested=true 2024-11-11T06:41:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:19,324 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:19,324 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d55fa98fb98bf91a607d5c6eb8d4ba7d:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:19,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:19,326 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 62086 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:19,326 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:19,326 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/A is initiating minor compaction (all files) 2024-11-11T06:41:19,326 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/B is initiating minor compaction (all files) 2024-11-11T06:41:19,326 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/A in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:19,326 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/B in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:19,326 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/86830409963e436ba8a1a9f86931db0b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e2ee49da487641668a90fb968f1065ea, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/146206a54be2467b8c7a05a2aefea181, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9b30cf237c2f4d1c8516645c5003d80a] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=60.6 K 2024-11-11T06:41:19,326 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0a2e52580d054104b0ab7d70fb50c2ed, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/44b0155bbbad4ff1b18fe74108e2bb7d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/5f57f4213eb1480c9591f329dba77aaf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4c0e2c91fa454b79958768b3b63406ed] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=48.7 K 2024-11-11T06:41:19,326 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a2e52580d054104b0ab7d70fb50c2ed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731307273389 2024-11-11T06:41:19,326 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86830409963e436ba8a1a9f86931db0b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731307273389 2024-11-11T06:41:19,327 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b0155bbbad4ff1b18fe74108e2bb7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731307274059 2024-11-11T06:41:19,327 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ee49da487641668a90fb968f1065ea, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731307274059 2024-11-11T06:41:19,327 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f57f4213eb1480c9591f329dba77aaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307275216 2024-11-11T06:41:19,327 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 146206a54be2467b8c7a05a2aefea181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307275216 2024-11-11T06:41:19,327 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c0e2c91fa454b79958768b3b63406ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731307278040 2024-11-11T06:41:19,328 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b30cf237c2f4d1c8516645c5003d80a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731307277408 2024-11-11T06:41:19,339 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#B#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:19,339 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/9dc9525b9a354bce8f588ac3c6f3c924 is 50, key is test_row_0/B:col10/1731307278040/Put/seqid=0 2024-11-11T06:41:19,340 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#A#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:19,340 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/c7390ddf3d184651999a426e02b2d1f2 is 50, key is test_row_0/A:col10/1731307278040/Put/seqid=0 2024-11-11T06:41:19,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742134_1310 (size=13119) 2024-11-11T06:41:19,362 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/c7390ddf3d184651999a426e02b2d1f2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/c7390ddf3d184651999a426e02b2d1f2 2024-11-11T06:41:19,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742135_1311 (size=13119) 2024-11-11T06:41:19,369 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/A of d55fa98fb98bf91a607d5c6eb8d4ba7d into c7390ddf3d184651999a426e02b2d1f2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:19,369 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:19,369 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/A, priority=12, startTime=1731307279324; duration=0sec 2024-11-11T06:41:19,369 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:19,369 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:A 2024-11-11T06:41:19,369 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:41:19,370 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:41:19,370 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): d55fa98fb98bf91a607d5c6eb8d4ba7d/C is initiating minor compaction (all files) 2024-11-11T06:41:19,370 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d55fa98fb98bf91a607d5c6eb8d4ba7d/C in TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:19,370 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5e8f8bb98f8245dba1a6cc667e317444, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/01174fc685d84d7387af8ed855dcbf07, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/ea1fc7b4b9bf48518c10ac7370accf7c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/53f551f5588946ac8151465cb6b59255] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp, totalSize=48.7 K 2024-11-11T06:41:19,370 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8f8bb98f8245dba1a6cc667e317444, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731307273389 2024-11-11T06:41:19,371 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01174fc685d84d7387af8ed855dcbf07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731307274059 2024-11-11T06:41:19,371 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea1fc7b4b9bf48518c10ac7370accf7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731307275216 2024-11-11T06:41:19,371 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53f551f5588946ac8151465cb6b59255, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731307278040 2024-11-11T06:41:19,396 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d55fa98fb98bf91a607d5c6eb8d4ba7d#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:19,396 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/301003c01cfb4e7eb1ced39d7e15dcd1 is 50, key is test_row_0/C:col10/1731307278040/Put/seqid=0 2024-11-11T06:41:19,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742136_1312 (size=13119) 2024-11-11T06:41:19,530 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-11T06:41:19,530 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:19,530 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,530 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:19,532 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-11T06:41:19,532 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:19,532 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,532 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:19,532 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-11T06:41:19,532 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:19,532 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,532 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:19,533 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-11T06:41:19,533 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:19,533 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,533 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,533 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:19,535 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-11T06:41:19,535 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:19,535 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,535 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:19,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T06:41:19,731 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:19,778 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/9dc9525b9a354bce8f588ac3c6f3c924 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/9dc9525b9a354bce8f588ac3c6f3c924 2024-11-11T06:41:19,783 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/B of d55fa98fb98bf91a607d5c6eb8d4ba7d into 9dc9525b9a354bce8f588ac3c6f3c924(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:19,783 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:19,783 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/B, priority=12, startTime=1731307279324; duration=0sec 2024-11-11T06:41:19,783 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:19,783 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:B 2024-11-11T06:41:19,836 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/301003c01cfb4e7eb1ced39d7e15dcd1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/301003c01cfb4e7eb1ced39d7e15dcd1 2024-11-11T06:41:19,842 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d55fa98fb98bf91a607d5c6eb8d4ba7d/C of d55fa98fb98bf91a607d5c6eb8d4ba7d into 301003c01cfb4e7eb1ced39d7e15dcd1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:19,842 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:19,842 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., storeName=d55fa98fb98bf91a607d5c6eb8d4ba7d/C, priority=12, startTime=1731307279324; duration=0sec 2024-11-11T06:41:19,842 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:19,842 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d55fa98fb98bf91a607d5c6eb8d4ba7d:C 2024-11-11T06:41:20,204 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-11T06:41:20,204 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:20,204 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:20,204 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:20,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:20,213 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:20,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:41:20,213 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-11T06:41:20,213 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:20,213 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:20,213 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-11T06:41:20,213 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:20,213 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:20,213 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:20,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:20,214 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:20,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:20,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/736ffff5f0b448449836ce9406fe2335 is 50, key is test_row_0/A:col10/1731307280211/Put/seqid=0 2024-11-11T06:41:20,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742137_1313 (size=9857) 2024-11-11T06:41:20,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/736ffff5f0b448449836ce9406fe2335 2024-11-11T06:41:20,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/80a128e6ef3b47038d87365119a65246 is 50, key is test_row_0/B:col10/1731307280211/Put/seqid=0 2024-11-11T06:41:20,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742138_1314 (size=9857) 2024-11-11T06:41:21,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/80a128e6ef3b47038d87365119a65246 2024-11-11T06:41:21,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0e2ee56138114156bcc6364b10f09f0b is 50, key is test_row_0/C:col10/1731307280211/Put/seqid=0 2024-11-11T06:41:21,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742139_1315 (size=9857) 2024-11-11T06:41:21,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0e2ee56138114156bcc6364b10f09f0b 2024-11-11T06:41:21,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/736ffff5f0b448449836ce9406fe2335 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/736ffff5f0b448449836ce9406fe2335 2024-11-11T06:41:21,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/736ffff5f0b448449836ce9406fe2335, entries=100, sequenceid=371, filesize=9.6 K 2024-11-11T06:41:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/80a128e6ef3b47038d87365119a65246 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/80a128e6ef3b47038d87365119a65246 2024-11-11T06:41:21,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/80a128e6ef3b47038d87365119a65246, entries=100, sequenceid=371, filesize=9.6 K 2024-11-11T06:41:21,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/0e2ee56138114156bcc6364b10f09f0b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0e2ee56138114156bcc6364b10f09f0b 2024-11-11T06:41:21,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0e2ee56138114156bcc6364b10f09f0b, entries=100, sequenceid=371, filesize=9.6 K 2024-11-11T06:41:21,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1279ms, sequenceid=371, compaction requested=false 2024-11-11T06:41:21,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: 2024-11-11T06:41:27,693 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-11T06:41:27,693 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:27,693 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:27,693 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:27,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:27,744 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-11T06:41:27,744 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:27,744 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:27,744 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:27,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4131 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4057 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4169 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4126 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4078 2024-11-11T06:41:27,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T06:41:27,746 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T06:41:27,746 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:41:27,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:27,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:27,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:27,748 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T06:41:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T06:41:27,752 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307287752"}]},"ts":"1731307287752"} 2024-11-11T06:41:27,754 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T06:41:27,754 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T06:41:27,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:41:27,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, UNASSIGN}] 2024-11-11T06:41:27,758 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, UNASSIGN 2024-11-11T06:41:27,759 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=d55fa98fb98bf91a607d5c6eb8d4ba7d, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:27,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, UNASSIGN because future has completed 2024-11-11T06:41:27,761 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:41:27,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:41:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T06:41:27,914 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:27,915 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:41:27,915 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing d55fa98fb98bf91a607d5c6eb8d4ba7d, disabling compactions & flushes 2024-11-11T06:41:27,915 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:27,915 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:27,915 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. after waiting 0 ms 2024-11-11T06:41:27,915 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:27,915 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-11T06:41:27,921 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/c307cdee25f644fc8d7b264e082798b5 is 50, key is test_row_2/A:col10/1731307287741/Put/seqid=0 2024-11-11T06:41:27,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742140_1316 (size=7415) 2024-11-11T06:41:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T06:41:28,327 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/c307cdee25f644fc8d7b264e082798b5 2024-11-11T06:41:28,333 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4d39f222326642509885f0e3386ddab7 is 50, key is test_row_2/B:col10/1731307287741/Put/seqid=0 2024-11-11T06:41:28,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742141_1317 (size=7415) 2024-11-11T06:41:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T06:41:28,738 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4d39f222326642509885f0e3386ddab7 2024-11-11T06:41:28,754 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/4e7673914ba44c8a8ba46100e0cdc42f is 50, key is test_row_2/C:col10/1731307287741/Put/seqid=0 2024-11-11T06:41:28,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742142_1318 (size=7415) 2024-11-11T06:41:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T06:41:29,160 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/4e7673914ba44c8a8ba46100e0cdc42f 2024-11-11T06:41:29,171 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/A/c307cdee25f644fc8d7b264e082798b5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/c307cdee25f644fc8d7b264e082798b5 2024-11-11T06:41:29,176 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/c307cdee25f644fc8d7b264e082798b5, entries=50, sequenceid=376, filesize=7.2 K 2024-11-11T06:41:29,177 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/B/4d39f222326642509885f0e3386ddab7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4d39f222326642509885f0e3386ddab7 2024-11-11T06:41:29,181 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4d39f222326642509885f0e3386ddab7, entries=50, sequenceid=376, filesize=7.2 K 2024-11-11T06:41:29,182 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/.tmp/C/4e7673914ba44c8a8ba46100e0cdc42f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4e7673914ba44c8a8ba46100e0cdc42f 2024-11-11T06:41:29,186 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4e7673914ba44c8a8ba46100e0cdc42f, entries=50, sequenceid=376, filesize=7.2 K 2024-11-11T06:41:29,187 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1272ms, sequenceid=376, compaction requested=true 2024-11-11T06:41:29,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2739718129ea411a9893f35e8939ee40, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/63de424bfbb940c5bee59f1a158ba400, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/7f2b21d55f0e4c5dbe509add244242ec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/3a6ca17b31fd4c33a9f85cf33970c361, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/8756ba76b9d743388082411feac57543, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b88d6e5b1e8f43b18dd6b890a09c8abf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/99aa456a35ae4f2cb7a6bd8b9955e987, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/f22790912cd04290ae8d7ae1a19452c5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9fc0de40bfcb4770b99f779c977b4d67, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/31b11bb961fe464e80090d2f445c23ed, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/bb81d4ac0f9b402289811283ee9b2087, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/cfc3c58fbe4a42ac8b2fb2cc1e7e6336, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e06c7a15e9b3443e809ac04d33448210, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/ee399e574f0b4966b1cafb37f7ece10a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b648d02189f14acdbe2652d56c71470e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/0998722ca1394a97bf041a6836bc957e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2d1b80cde1924badb02b87cd53c76230, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/454373779d354494a225428b4504322d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/173aa155bccc44ef889063023781bc2a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/67f02882f9174a8a92285c95b193a1c0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/86830409963e436ba8a1a9f86931db0b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e2ee49da487641668a90fb968f1065ea, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/146206a54be2467b8c7a05a2aefea181, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9b30cf237c2f4d1c8516645c5003d80a] to archive 2024-11-11T06:41:29,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:41:29,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2739718129ea411a9893f35e8939ee40 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2739718129ea411a9893f35e8939ee40 2024-11-11T06:41:29,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/63de424bfbb940c5bee59f1a158ba400 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/63de424bfbb940c5bee59f1a158ba400 2024-11-11T06:41:29,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/7f2b21d55f0e4c5dbe509add244242ec to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/7f2b21d55f0e4c5dbe509add244242ec 2024-11-11T06:41:29,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/3a6ca17b31fd4c33a9f85cf33970c361 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/3a6ca17b31fd4c33a9f85cf33970c361 2024-11-11T06:41:29,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/8756ba76b9d743388082411feac57543 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/8756ba76b9d743388082411feac57543 2024-11-11T06:41:29,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b88d6e5b1e8f43b18dd6b890a09c8abf to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b88d6e5b1e8f43b18dd6b890a09c8abf 2024-11-11T06:41:29,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/99aa456a35ae4f2cb7a6bd8b9955e987 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/99aa456a35ae4f2cb7a6bd8b9955e987 2024-11-11T06:41:29,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/f22790912cd04290ae8d7ae1a19452c5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/f22790912cd04290ae8d7ae1a19452c5 2024-11-11T06:41:29,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9fc0de40bfcb4770b99f779c977b4d67 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9fc0de40bfcb4770b99f779c977b4d67 2024-11-11T06:41:29,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/31b11bb961fe464e80090d2f445c23ed to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/31b11bb961fe464e80090d2f445c23ed 2024-11-11T06:41:29,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/bb81d4ac0f9b402289811283ee9b2087 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/bb81d4ac0f9b402289811283ee9b2087 2024-11-11T06:41:29,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/cfc3c58fbe4a42ac8b2fb2cc1e7e6336 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/cfc3c58fbe4a42ac8b2fb2cc1e7e6336 2024-11-11T06:41:29,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e06c7a15e9b3443e809ac04d33448210 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e06c7a15e9b3443e809ac04d33448210 2024-11-11T06:41:29,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/ee399e574f0b4966b1cafb37f7ece10a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/ee399e574f0b4966b1cafb37f7ece10a 2024-11-11T06:41:29,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b648d02189f14acdbe2652d56c71470e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/b648d02189f14acdbe2652d56c71470e 2024-11-11T06:41:29,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/0998722ca1394a97bf041a6836bc957e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/0998722ca1394a97bf041a6836bc957e 2024-11-11T06:41:29,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2d1b80cde1924badb02b87cd53c76230 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/2d1b80cde1924badb02b87cd53c76230 2024-11-11T06:41:29,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/454373779d354494a225428b4504322d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/454373779d354494a225428b4504322d 2024-11-11T06:41:29,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/173aa155bccc44ef889063023781bc2a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/173aa155bccc44ef889063023781bc2a 2024-11-11T06:41:29,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/67f02882f9174a8a92285c95b193a1c0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/67f02882f9174a8a92285c95b193a1c0 2024-11-11T06:41:29,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/86830409963e436ba8a1a9f86931db0b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/86830409963e436ba8a1a9f86931db0b 2024-11-11T06:41:29,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e2ee49da487641668a90fb968f1065ea to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/e2ee49da487641668a90fb968f1065ea 2024-11-11T06:41:29,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/146206a54be2467b8c7a05a2aefea181 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/146206a54be2467b8c7a05a2aefea181 2024-11-11T06:41:29,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9b30cf237c2f4d1c8516645c5003d80a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/9b30cf237c2f4d1c8516645c5003d80a 2024-11-11T06:41:29,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/ab86d38840ae4639b8605ce343793e66, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6b3a9554c7fd45d39d66621c71a8ab7b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/41795dc0841943ea9b22ccce7fea4c67, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/17af9d0d74c448178d7b4fd09d281d23, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/99b20b39fc0945868ced806c85eccb95, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6fd0f6f559f94632a6fff1e6a80c961e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2ff1c6cff57345f6afcbfc17cc92a9ba, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2b29c1ab53634f0dae62372dcdbd08bd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/d7e9976a1c914f588d2391f547dd1a85, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/461072d74bf54c8fb5e21ab11ee434e8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4cc289a251da4dd990f69b93847a566f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0543b39fa7d94b75ba7b335058ac447d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/92f514151fd5409ab58ca0d7a926bd44, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/3f73f3b935104f8d86d3f57e660a16ab, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/bfc540b99b744482a51a89dbf9ed2975, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b21d9f8a67204dbaa7eeff3e9a90aad2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/57cbcdab351e49689d9d42c98a138e18, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/c07e2491ea7646c3881598b35de847f6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0a2e52580d054104b0ab7d70fb50c2ed, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/852f1efd9c70427b9596d3c6c600a1d3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/44b0155bbbad4ff1b18fe74108e2bb7d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/5f57f4213eb1480c9591f329dba77aaf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4c0e2c91fa454b79958768b3b63406ed] to archive 2024-11-11T06:41:29,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:41:29,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/ab86d38840ae4639b8605ce343793e66 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/ab86d38840ae4639b8605ce343793e66 2024-11-11T06:41:29,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b1bfd6ddb96d4c63b2820e2f4a02b9f8 2024-11-11T06:41:29,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6b3a9554c7fd45d39d66621c71a8ab7b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6b3a9554c7fd45d39d66621c71a8ab7b 2024-11-11T06:41:29,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/41795dc0841943ea9b22ccce7fea4c67 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/41795dc0841943ea9b22ccce7fea4c67 2024-11-11T06:41:29,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/17af9d0d74c448178d7b4fd09d281d23 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/17af9d0d74c448178d7b4fd09d281d23 2024-11-11T06:41:29,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/99b20b39fc0945868ced806c85eccb95 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/99b20b39fc0945868ced806c85eccb95 2024-11-11T06:41:29,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6fd0f6f559f94632a6fff1e6a80c961e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/6fd0f6f559f94632a6fff1e6a80c961e 2024-11-11T06:41:29,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2ff1c6cff57345f6afcbfc17cc92a9ba to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2ff1c6cff57345f6afcbfc17cc92a9ba 2024-11-11T06:41:29,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2b29c1ab53634f0dae62372dcdbd08bd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/2b29c1ab53634f0dae62372dcdbd08bd 2024-11-11T06:41:29,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/d7e9976a1c914f588d2391f547dd1a85 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/d7e9976a1c914f588d2391f547dd1a85 2024-11-11T06:41:29,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/461072d74bf54c8fb5e21ab11ee434e8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/461072d74bf54c8fb5e21ab11ee434e8 2024-11-11T06:41:29,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4cc289a251da4dd990f69b93847a566f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4cc289a251da4dd990f69b93847a566f 2024-11-11T06:41:29,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0543b39fa7d94b75ba7b335058ac447d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0543b39fa7d94b75ba7b335058ac447d 2024-11-11T06:41:29,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/92f514151fd5409ab58ca0d7a926bd44 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/92f514151fd5409ab58ca0d7a926bd44 2024-11-11T06:41:29,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/3f73f3b935104f8d86d3f57e660a16ab to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/3f73f3b935104f8d86d3f57e660a16ab 2024-11-11T06:41:29,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/bfc540b99b744482a51a89dbf9ed2975 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/bfc540b99b744482a51a89dbf9ed2975 2024-11-11T06:41:29,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b21d9f8a67204dbaa7eeff3e9a90aad2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/b21d9f8a67204dbaa7eeff3e9a90aad2 2024-11-11T06:41:29,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/57cbcdab351e49689d9d42c98a138e18 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/57cbcdab351e49689d9d42c98a138e18 2024-11-11T06:41:29,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/c07e2491ea7646c3881598b35de847f6 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/c07e2491ea7646c3881598b35de847f6 2024-11-11T06:41:29,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0a2e52580d054104b0ab7d70fb50c2ed to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/0a2e52580d054104b0ab7d70fb50c2ed 2024-11-11T06:41:29,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/852f1efd9c70427b9596d3c6c600a1d3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/852f1efd9c70427b9596d3c6c600a1d3 2024-11-11T06:41:29,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/44b0155bbbad4ff1b18fe74108e2bb7d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/44b0155bbbad4ff1b18fe74108e2bb7d 2024-11-11T06:41:29,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/5f57f4213eb1480c9591f329dba77aaf to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/5f57f4213eb1480c9591f329dba77aaf 2024-11-11T06:41:29,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4c0e2c91fa454b79958768b3b63406ed to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4c0e2c91fa454b79958768b3b63406ed 2024-11-11T06:41:29,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5484da26a67c4173b2ea6a54b5d1ba36, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a816080ed9f54ac78b0e91e981737d1a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/2317c056220340da89ffab8b64ab953b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/8b723c338b654314a8c503c83391369a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e7cc5c8cb703461ca458f212fb0c773d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/defa83820f4342cf84d0d0d51da827d8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/febc77c0ca1a44cdb54a4ce88481e7f3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0f84a849b2ef4f0198346f69d682fbe4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e02272632afa4f848296487b903504a5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0bc20ca6d28c4d02b754a7ada047e472, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4d83d3e2d81f498ea3c3b350b05569cd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a3be913495ae4cefa66e6e2e8b5c6a0c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/f6de3281198d4d298154daae25a66801, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/48cc7df465094d6abf48c4cfd667fc8f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/74d763a6c04b4ac1affb56b02dc01292, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4a215fc814b54d0ba381d3eb4c507d4d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/3a4f1223835542048a51a31c64dea719, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/94308e43657a4ba3b1298c8fa9862e6a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/05677989c4144179a1f26c61f25b4890, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5e8f8bb98f8245dba1a6cc667e317444, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/7d7cf471f8124bbaa70e824918308c52, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/01174fc685d84d7387af8ed855dcbf07, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/ea1fc7b4b9bf48518c10ac7370accf7c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/53f551f5588946ac8151465cb6b59255] to archive 2024-11-11T06:41:29,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:41:29,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5484da26a67c4173b2ea6a54b5d1ba36 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5484da26a67c4173b2ea6a54b5d1ba36 2024-11-11T06:41:29,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a816080ed9f54ac78b0e91e981737d1a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a816080ed9f54ac78b0e91e981737d1a 2024-11-11T06:41:29,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/2317c056220340da89ffab8b64ab953b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/2317c056220340da89ffab8b64ab953b 2024-11-11T06:41:29,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/8b723c338b654314a8c503c83391369a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/8b723c338b654314a8c503c83391369a 2024-11-11T06:41:29,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e7cc5c8cb703461ca458f212fb0c773d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e7cc5c8cb703461ca458f212fb0c773d 2024-11-11T06:41:29,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/defa83820f4342cf84d0d0d51da827d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/defa83820f4342cf84d0d0d51da827d8 2024-11-11T06:41:29,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/febc77c0ca1a44cdb54a4ce88481e7f3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/febc77c0ca1a44cdb54a4ce88481e7f3 2024-11-11T06:41:29,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0f84a849b2ef4f0198346f69d682fbe4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0f84a849b2ef4f0198346f69d682fbe4 2024-11-11T06:41:29,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e02272632afa4f848296487b903504a5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/e02272632afa4f848296487b903504a5 2024-11-11T06:41:29,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0bc20ca6d28c4d02b754a7ada047e472 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0bc20ca6d28c4d02b754a7ada047e472 2024-11-11T06:41:29,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4d83d3e2d81f498ea3c3b350b05569cd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4d83d3e2d81f498ea3c3b350b05569cd 2024-11-11T06:41:29,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a3be913495ae4cefa66e6e2e8b5c6a0c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/a3be913495ae4cefa66e6e2e8b5c6a0c 2024-11-11T06:41:29,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/f6de3281198d4d298154daae25a66801 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/f6de3281198d4d298154daae25a66801 2024-11-11T06:41:29,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/48cc7df465094d6abf48c4cfd667fc8f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/48cc7df465094d6abf48c4cfd667fc8f 2024-11-11T06:41:29,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/74d763a6c04b4ac1affb56b02dc01292 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/74d763a6c04b4ac1affb56b02dc01292 2024-11-11T06:41:29,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4a215fc814b54d0ba381d3eb4c507d4d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4a215fc814b54d0ba381d3eb4c507d4d 2024-11-11T06:41:29,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/3a4f1223835542048a51a31c64dea719 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/3a4f1223835542048a51a31c64dea719 2024-11-11T06:41:29,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/94308e43657a4ba3b1298c8fa9862e6a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/94308e43657a4ba3b1298c8fa9862e6a 2024-11-11T06:41:29,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/05677989c4144179a1f26c61f25b4890 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/05677989c4144179a1f26c61f25b4890 2024-11-11T06:41:29,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5e8f8bb98f8245dba1a6cc667e317444 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/5e8f8bb98f8245dba1a6cc667e317444 2024-11-11T06:41:29,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/7d7cf471f8124bbaa70e824918308c52 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/7d7cf471f8124bbaa70e824918308c52 2024-11-11T06:41:29,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/01174fc685d84d7387af8ed855dcbf07 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/01174fc685d84d7387af8ed855dcbf07 2024-11-11T06:41:29,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/ea1fc7b4b9bf48518c10ac7370accf7c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/ea1fc7b4b9bf48518c10ac7370accf7c 2024-11-11T06:41:29,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/53f551f5588946ac8151465cb6b59255 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/53f551f5588946ac8151465cb6b59255 2024-11-11T06:41:29,261 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/recovered.edits/379.seqid, newMaxSeqId=379, maxSeqId=1 2024-11-11T06:41:29,262 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. 2024-11-11T06:41:29,262 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for d55fa98fb98bf91a607d5c6eb8d4ba7d: Waiting for close lock at 1731307287915Running coprocessor pre-close hooks at 1731307287915Disabling compacts and flushes for region at 1731307287915Disabling writes for close at 1731307287915Obtaining lock to block concurrent updates at 1731307287915Preparing flush snapshotting stores in d55fa98fb98bf91a607d5c6eb8d4ba7d at 1731307287915Finished memstore snapshotting TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731307287915Flushing stores of TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d. at 1731307287916 (+1 ms)Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/A: creating writer at 1731307287916Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/A: appending metadata at 1731307287920 (+4 ms)Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/A: closing flushed file at 1731307287920Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/B: creating writer at 1731307288330 (+410 ms)Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/B: appending metadata at 1731307288332 (+2 ms)Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/B: closing flushed file at 1731307288332Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/C: creating writer at 1731307288748 (+416 ms)Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/C: appending metadata at 1731307288753 (+5 ms)Flushing d55fa98fb98bf91a607d5c6eb8d4ba7d/C: closing flushed file at 1731307288753Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15fe765b: reopening flushed file at 1731307289170 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26ef68a2: reopening flushed file at 1731307289176 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fa0ed58: reopening flushed file at 1731307289181 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for d55fa98fb98bf91a607d5c6eb8d4ba7d in 1272ms, sequenceid=376, compaction requested=true at 1731307289187 (+6 ms)Writing region close event to WAL at 1731307289259 (+72 ms)Running coprocessor post-close hooks at 1731307289262 (+3 ms)Closed at 1731307289262 2024-11-11T06:41:29,264 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:29,264 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=d55fa98fb98bf91a607d5c6eb8d4ba7d, regionState=CLOSED 2024-11-11T06:41:29,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:41:29,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T06:41:29,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure d55fa98fb98bf91a607d5c6eb8d4ba7d, server=ce2cf0244921,35807,1731307183669 in 1.5050 sec 2024-11-11T06:41:29,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-11T06:41:29,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d55fa98fb98bf91a607d5c6eb8d4ba7d, UNASSIGN in 1.5100 sec 2024-11-11T06:41:29,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T06:41:29,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5140 sec 2024-11-11T06:41:29,271 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307289271"}]},"ts":"1731307289271"} 2024-11-11T06:41:29,272 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T06:41:29,272 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T06:41:29,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5250 sec 2024-11-11T06:41:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T06:41:29,900 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:29,902 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T06:41:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:29,908 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T06:41:29,910 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:29,913 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:29,916 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/recovered.edits] 2024-11-11T06:41:29,919 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/736ffff5f0b448449836ce9406fe2335 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/736ffff5f0b448449836ce9406fe2335 2024-11-11T06:41:29,921 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/c307cdee25f644fc8d7b264e082798b5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/c307cdee25f644fc8d7b264e082798b5 2024-11-11T06:41:29,922 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/c7390ddf3d184651999a426e02b2d1f2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/A/c7390ddf3d184651999a426e02b2d1f2 2024-11-11T06:41:29,925 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4d39f222326642509885f0e3386ddab7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/4d39f222326642509885f0e3386ddab7 2024-11-11T06:41:29,926 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/80a128e6ef3b47038d87365119a65246 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/80a128e6ef3b47038d87365119a65246 2024-11-11T06:41:29,928 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/9dc9525b9a354bce8f588ac3c6f3c924 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/B/9dc9525b9a354bce8f588ac3c6f3c924 2024-11-11T06:41:29,931 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0e2ee56138114156bcc6364b10f09f0b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/0e2ee56138114156bcc6364b10f09f0b 2024-11-11T06:41:29,932 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/301003c01cfb4e7eb1ced39d7e15dcd1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/301003c01cfb4e7eb1ced39d7e15dcd1 2024-11-11T06:41:29,934 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4e7673914ba44c8a8ba46100e0cdc42f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/C/4e7673914ba44c8a8ba46100e0cdc42f 2024-11-11T06:41:29,937 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/recovered.edits/379.seqid to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d/recovered.edits/379.seqid 2024-11-11T06:41:29,937 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/d55fa98fb98bf91a607d5c6eb8d4ba7d 2024-11-11T06:41:29,937 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T06:41:29,940 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:29,944 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T06:41:29,947 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T06:41:29,948 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:29,948 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T06:41:29,949 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731307289949"}]},"ts":"9223372036854775807"} 2024-11-11T06:41:29,951 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T06:41:29,951 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d55fa98fb98bf91a607d5c6eb8d4ba7d, NAME => 'TestAcidGuarantees,,1731307258317.d55fa98fb98bf91a607d5c6eb8d4ba7d.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T06:41:29,951 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T06:41:29,951 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731307289951"}]},"ts":"9223372036854775807"} 2024-11-11T06:41:29,952 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T06:41:29,953 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:29,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-11T06:41:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T06:41:30,020 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:30,028 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237 (was 238), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=423 (was 479), ProcessCount=11 (was 11), AvailableMemoryMB=10929 (was 11063) 2024-11-11T06:41:30,035 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=423, ProcessCount=11, AvailableMemoryMB=10929 2024-11-11T06:41:30,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:41:30,037 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:41:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:30,038 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T06:41:30,038 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:30,039 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-11T06:41:30,039 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T06:41:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T06:41:30,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742143_1319 (size=959) 2024-11-11T06:41:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T06:41:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T06:41:30,450 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 2024-11-11T06:41:30,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742144_1320 (size=53) 2024-11-11T06:41:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T06:41:30,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:41:30,862 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1a604f2419334e3a9e7d9b217f70c4d8, disabling compactions & flushes 2024-11-11T06:41:30,862 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:30,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:30,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. after waiting 0 ms 2024-11-11T06:41:30,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:30,863 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:30,863 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a604f2419334e3a9e7d9b217f70c4d8: Waiting for close lock at 1731307290862Disabling compacts and flushes for region at 1731307290862Disabling writes for close at 1731307290863 (+1 ms)Writing region close event to WAL at 1731307290863Closed at 1731307290863 2024-11-11T06:41:30,866 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T06:41:30,867 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731307290866"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731307290866"}]},"ts":"1731307290866"} 2024-11-11T06:41:30,870 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T06:41:30,871 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T06:41:30,872 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307290871"}]},"ts":"1731307290871"} 2024-11-11T06:41:30,873 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T06:41:30,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, ASSIGN}] 2024-11-11T06:41:30,874 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, ASSIGN 2024-11-11T06:41:30,875 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, ASSIGN; state=OFFLINE, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=false 2024-11-11T06:41:31,026 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=1a604f2419334e3a9e7d9b217f70c4d8, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:31,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, ASSIGN because future has completed 2024-11-11T06:41:31,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:41:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T06:41:31,190 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:31,191 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:41:31,191 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,191 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:41:31,191 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,191 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,194 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,196 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a604f2419334e3a9e7d9b217f70c4d8 columnFamilyName A 2024-11-11T06:41:31,196 DEBUG [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:31,197 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(327): Store=1a604f2419334e3a9e7d9b217f70c4d8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:41:31,197 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,199 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a604f2419334e3a9e7d9b217f70c4d8 columnFamilyName B 2024-11-11T06:41:31,199 DEBUG [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:31,200 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(327): Store=1a604f2419334e3a9e7d9b217f70c4d8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:41:31,200 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,202 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a604f2419334e3a9e7d9b217f70c4d8 columnFamilyName C 2024-11-11T06:41:31,202 DEBUG [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:31,203 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(327): Store=1a604f2419334e3a9e7d9b217f70c4d8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:41:31,203 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,204 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,204 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,205 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,205 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,206 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:41:31,208 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,210 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T06:41:31,210 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 1a604f2419334e3a9e7d9b217f70c4d8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64257862, jitterRate=-0.042483240365982056}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:41:31,211 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:31,211 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 1a604f2419334e3a9e7d9b217f70c4d8: Running coprocessor pre-open hook at 1731307291192Writing region info on filesystem at 1731307291192Initializing all the Stores at 1731307291193 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307291193Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307291194 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307291194Cleaning up temporary data from old regions at 1731307291205 (+11 ms)Running coprocessor post-open hooks at 1731307291211 (+6 ms)Region opened successfully at 1731307291211 2024-11-11T06:41:31,212 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., pid=89, masterSystemTime=1731307291183 2024-11-11T06:41:31,214 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:31,214 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:31,215 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=1a604f2419334e3a9e7d9b217f70c4d8, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:31,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:41:31,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T06:41:31,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 in 187 msec 2024-11-11T06:41:31,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T06:41:31,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, ASSIGN in 345 msec 2024-11-11T06:41:31,220 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T06:41:31,220 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307291220"}]},"ts":"1731307291220"} 2024-11-11T06:41:31,221 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T06:41:31,222 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T06:41:31,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-11T06:41:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T06:41:32,200 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:32,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:32,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:32,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:32,206 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f807ee7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:32,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:32,211 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6492d6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:32,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:32,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:32,214 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T06:41:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,40845,1731307182847 2024-11-11T06:41:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c02faa5 2024-11-11T06:41:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T06:41:32,216 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T06:41:32,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:41:32,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:41:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T06:41:32,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742145_1321 (size=995) 2024-11-11T06:41:32,683 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T06:41:32,683 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T06:41:32,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:41:32,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, REOPEN/MOVE}] 2024-11-11T06:41:32,691 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, REOPEN/MOVE 2024-11-11T06:41:32,692 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=1a604f2419334e3a9e7d9b217f70c4d8, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:32,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, REOPEN/MOVE because future has completed 2024-11-11T06:41:32,693 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:41:32,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:41:32,847 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:32,847 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:41:32,848 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 1a604f2419334e3a9e7d9b217f70c4d8, disabling compactions & flushes 2024-11-11T06:41:32,848 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:32,848 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:32,848 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. after waiting 0 ms 2024-11-11T06:41:32,848 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:32,857 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T06:41:32,859 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:32,859 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 1a604f2419334e3a9e7d9b217f70c4d8: Waiting for close lock at 1731307292847Running coprocessor pre-close hooks at 1731307292847Disabling compacts and flushes for region at 1731307292847Disabling writes for close at 1731307292848 (+1 ms)Writing region close event to WAL at 1731307292849 (+1 ms)Running coprocessor post-close hooks at 1731307292858 (+9 ms)Closed at 1731307292858 2024-11-11T06:41:32,859 WARN [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 1a604f2419334e3a9e7d9b217f70c4d8 to self. 2024-11-11T06:41:32,862 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:32,863 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=1a604f2419334e3a9e7d9b217f70c4d8, regionState=CLOSED 2024-11-11T06:41:32,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:41:32,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-11T06:41:32,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 in 173 msec 2024-11-11T06:41:32,869 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, REOPEN/MOVE; state=CLOSED, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=true 2024-11-11T06:41:33,020 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=1a604f2419334e3a9e7d9b217f70c4d8, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, REOPEN/MOVE because future has completed 2024-11-11T06:41:33,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:41:33,187 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,187 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:41:33,187 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,188 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:41:33,188 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,188 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,190 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,191 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a604f2419334e3a9e7d9b217f70c4d8 columnFamilyName A 2024-11-11T06:41:33,192 DEBUG [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:33,192 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(327): Store=1a604f2419334e3a9e7d9b217f70c4d8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:41:33,193 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,193 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a604f2419334e3a9e7d9b217f70c4d8 columnFamilyName B 2024-11-11T06:41:33,194 DEBUG [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:33,194 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(327): Store=1a604f2419334e3a9e7d9b217f70c4d8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:41:33,194 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,195 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a604f2419334e3a9e7d9b217f70c4d8 columnFamilyName C 2024-11-11T06:41:33,195 DEBUG [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:33,195 INFO [StoreOpener-1a604f2419334e3a9e7d9b217f70c4d8-1 {}] regionserver.HStore(327): Store=1a604f2419334e3a9e7d9b217f70c4d8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:41:33,195 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,196 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,196 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,197 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,197 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,198 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:41:33,199 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,199 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 1a604f2419334e3a9e7d9b217f70c4d8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73492564, jitterRate=0.09512454271316528}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:41:33,199 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,200 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 1a604f2419334e3a9e7d9b217f70c4d8: Running coprocessor pre-open hook at 1731307293188Writing region info on filesystem at 1731307293188Initializing all the Stores at 1731307293189 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307293189Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307293190 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307293190Cleaning up temporary data from old regions at 1731307293197 (+7 ms)Running coprocessor post-open hooks at 1731307293199 (+2 ms)Region opened successfully at 1731307293200 (+1 ms) 2024-11-11T06:41:33,201 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., pid=94, masterSystemTime=1731307293179 2024-11-11T06:41:33,202 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,202 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,203 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=1a604f2419334e3a9e7d9b217f70c4d8, regionState=OPEN, openSeqNum=5, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:41:33,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-11T06:41:33,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 in 180 msec 2024-11-11T06:41:33,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T06:41:33,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, REOPEN/MOVE in 516 msec 2024-11-11T06:41:33,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-11T06:41:33,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-11T06:41:33,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 992 msec 2024-11-11T06:41:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T06:41:33,212 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:33,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d1bdb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbfe2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,216 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@248132e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae968b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,218 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ecb65a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,220 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c5f763, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,221 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e91250f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,223 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c447bee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,224 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37577ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,225 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bac3357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,227 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14da6c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,228 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422cc54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,230 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ad3550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,231 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@423aef86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,233 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e0d290, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,234 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c9b05d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c421a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,237 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff4977c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:41:33,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:41:33,238 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:41:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:41:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:41:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2deadd1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:41:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:41:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:33,240 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:41:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bcf56fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:41:33,240 DEBUG [Thread-1480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,240 DEBUG [Thread-1484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,241 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,241 DEBUG [Thread-1493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,241 DEBUG [Thread-1488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,241 DEBUG [Thread-1497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,242 DEBUG [Thread-1491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,242 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T06:41:33,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T06:41:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,246 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:33,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,247 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:33,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,251 DEBUG [Thread-1495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,251 DEBUG [Thread-1486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,251 DEBUG [Thread-1482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:41:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:41:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:41:33,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:41:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:41:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,258 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:33,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:41:33,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d0c5a4aaf45b419eb25a6d57f16031a0_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307293257/Put/seqid=0 2024-11-11T06:41:33,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742146_1322 (size=12154) 2024-11-11T06:41:33,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:33,303 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d0c5a4aaf45b419eb25a6d57f16031a0_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d0c5a4aaf45b419eb25a6d57f16031a0_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,304 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/aea5e1d287b242bc820e41e4e58c3a26, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:33,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/aea5e1d287b242bc820e41e4e58c3a26 is 175, key is test_row_0/A:col10/1731307293257/Put/seqid=0 2024-11-11T06:41:33,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307353297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307353298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307353301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307353301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307353307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742147_1323 (size=30955) 2024-11-11T06:41:33,319 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/aea5e1d287b242bc820e41e4e58c3a26 2024-11-11T06:41:33,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/748acd342e2f4e5ea4eb015db37535bc is 50, key is test_row_0/B:col10/1731307293257/Put/seqid=0 2024-11-11T06:41:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T06:41:33,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742148_1324 (size=12001) 2024-11-11T06:41:33,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/748acd342e2f4e5ea4eb015db37535bc 2024-11-11T06:41:33,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/59bdc12f30954bab8f5fbd3f5c602c88 is 50, key is test_row_0/C:col10/1731307293257/Put/seqid=0 2024-11-11T06:41:33,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742149_1325 (size=12001) 2024-11-11T06:41:33,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T06:41:33,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:33,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307353419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307353419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307353420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307353420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307353420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T06:41:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:33,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T06:41:33,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307353628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307353629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307353629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307353629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307353639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T06:41:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:33,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/59bdc12f30954bab8f5fbd3f5c602c88 2024-11-11T06:41:33,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/aea5e1d287b242bc820e41e4e58c3a26 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/aea5e1d287b242bc820e41e4e58c3a26 2024-11-11T06:41:33,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/aea5e1d287b242bc820e41e4e58c3a26, entries=150, sequenceid=15, filesize=30.2 K 2024-11-11T06:41:33,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/748acd342e2f4e5ea4eb015db37535bc as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/748acd342e2f4e5ea4eb015db37535bc 2024-11-11T06:41:33,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/748acd342e2f4e5ea4eb015db37535bc, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T06:41:33,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/59bdc12f30954bab8f5fbd3f5c602c88 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/59bdc12f30954bab8f5fbd3f5c602c88 2024-11-11T06:41:33,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/59bdc12f30954bab8f5fbd3f5c602c88, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T06:41:33,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1a604f2419334e3a9e7d9b217f70c4d8 in 535ms, sequenceid=15, compaction requested=false 2024-11-11T06:41:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:33,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T06:41:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:33,859 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:41:33,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cc23ec8cc70e4a5bb92cb739b4bbaf68_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307293299/Put/seqid=0 2024-11-11T06:41:33,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742150_1326 (size=12154) 2024-11-11T06:41:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T06:41:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:33,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307353947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307353949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307353952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307353952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:33,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307353953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307354059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307354059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307354059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307354069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,272 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cc23ec8cc70e4a5bb92cb739b4bbaf68_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc23ec8cc70e4a5bb92cb739b4bbaf68_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307354269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307354269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/b918d529df9f449fa954cccdd2bfc716, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307354269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/b918d529df9f449fa954cccdd2bfc716 is 175, key is test_row_0/A:col10/1731307293299/Put/seqid=0 2024-11-11T06:41:34,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307354279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742151_1327 (size=30955) 2024-11-11T06:41:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T06:41:34,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307354459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307354579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307354579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307354579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307354589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:34,689 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/b918d529df9f449fa954cccdd2bfc716 2024-11-11T06:41:34,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/555e2b65f40e49a8b6fc1cd9a3c7e089 is 50, key is test_row_0/B:col10/1731307293299/Put/seqid=0 2024-11-11T06:41:34,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742152_1328 (size=12001) 2024-11-11T06:41:34,700 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/555e2b65f40e49a8b6fc1cd9a3c7e089 2024-11-11T06:41:34,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/170ec40573de49f396bed16936260ae7 is 50, key is test_row_0/C:col10/1731307293299/Put/seqid=0 2024-11-11T06:41:34,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742153_1329 (size=12001) 2024-11-11T06:41:34,721 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/170ec40573de49f396bed16936260ae7 2024-11-11T06:41:34,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/b918d529df9f449fa954cccdd2bfc716 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/b918d529df9f449fa954cccdd2bfc716 2024-11-11T06:41:34,733 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/b918d529df9f449fa954cccdd2bfc716, entries=150, sequenceid=40, filesize=30.2 K 2024-11-11T06:41:34,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/555e2b65f40e49a8b6fc1cd9a3c7e089 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/555e2b65f40e49a8b6fc1cd9a3c7e089 2024-11-11T06:41:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,738 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/555e2b65f40e49a8b6fc1cd9a3c7e089, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T06:41:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/170ec40573de49f396bed16936260ae7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/170ec40573de49f396bed16936260ae7 2024-11-11T06:41:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,743 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/170ec40573de49f396bed16936260ae7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T06:41:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,744 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1a604f2419334e3a9e7d9b217f70c4d8 in 885ms, sequenceid=40, compaction requested=false 2024-11-11T06:41:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T06:41:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T06:41:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T06:41:34,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4980 sec 2024-11-11T06:41:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.5050 sec 2024-11-11T06:41:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,787 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T06:41:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:35,099 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:35,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:41:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d52213f141504dff91b9524a57f02111_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307293952/Put/seqid=0 2024-11-11T06:41:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742154_1330 (size=12154) 2024-11-11T06:41:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,118 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d52213f141504dff91b9524a57f02111_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d52213f141504dff91b9524a57f02111_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,124 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/13eab3af82be494db370fedeb4b7ab1c, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/13eab3af82be494db370fedeb4b7ab1c is 175, key is test_row_0/A:col10/1731307293952/Put/seqid=0 2024-11-11T06:41:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742155_1331 (size=30955) 2024-11-11T06:41:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/13eab3af82be494db370fedeb4b7ab1c 2024-11-11T06:41:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:35,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/28d373bba576471b8f07b6bb95d95010 is 50, key is test_row_0/B:col10/1731307293952/Put/seqid=0 2024-11-11T06:41:35,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742156_1332 (size=12001) 2024-11-11T06:41:35,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307355170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307355171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307355172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307355174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307355289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307355289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307355289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307355289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T06:41:35,390 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:35,390 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T06:41:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T06:41:35,391 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:35,392 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:35,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:35,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307355469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,473 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1a604f2419334e3a9e7d9b217f70c4d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T06:41:35,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307355498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307355509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307355509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307355509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/28d373bba576471b8f07b6bb95d95010 2024-11-11T06:41:35,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T06:41:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:35,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:35,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:35,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/04a6aa8212f54acc9cc73e1475797f27 is 50, key is test_row_0/C:col10/1731307293952/Put/seqid=0 2024-11-11T06:41:35,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742157_1333 (size=12001) 2024-11-11T06:41:35,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/04a6aa8212f54acc9cc73e1475797f27 2024-11-11T06:41:35,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/13eab3af82be494db370fedeb4b7ab1c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/13eab3af82be494db370fedeb4b7ab1c 2024-11-11T06:41:35,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/13eab3af82be494db370fedeb4b7ab1c, entries=150, sequenceid=52, filesize=30.2 K 2024-11-11T06:41:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/28d373bba576471b8f07b6bb95d95010 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/28d373bba576471b8f07b6bb95d95010 2024-11-11T06:41:35,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/28d373bba576471b8f07b6bb95d95010, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T06:41:35,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/04a6aa8212f54acc9cc73e1475797f27 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/04a6aa8212f54acc9cc73e1475797f27 2024-11-11T06:41:35,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/04a6aa8212f54acc9cc73e1475797f27, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T06:41:35,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1a604f2419334e3a9e7d9b217f70c4d8 in 483ms, sequenceid=52, compaction requested=true 2024-11-11T06:41:35,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:35,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:35,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:35,583 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:35,583 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:35,585 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:35,585 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:35,585 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/B is initiating minor compaction (all files) 2024-11-11T06:41:35,585 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/A is initiating minor compaction (all files) 2024-11-11T06:41:35,585 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/A in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:35,585 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/B in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:35,585 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/748acd342e2f4e5ea4eb015db37535bc, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/555e2b65f40e49a8b6fc1cd9a3c7e089, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/28d373bba576471b8f07b6bb95d95010] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.2 K 2024-11-11T06:41:35,585 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/aea5e1d287b242bc820e41e4e58c3a26, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/b918d529df9f449fa954cccdd2bfc716, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/13eab3af82be494db370fedeb4b7ab1c] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=90.7 K 2024-11-11T06:41:35,585 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:35,586 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/aea5e1d287b242bc820e41e4e58c3a26, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/b918d529df9f449fa954cccdd2bfc716, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/13eab3af82be494db370fedeb4b7ab1c] 2024-11-11T06:41:35,586 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 748acd342e2f4e5ea4eb015db37535bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731307293246 2024-11-11T06:41:35,586 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea5e1d287b242bc820e41e4e58c3a26, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731307293246 2024-11-11T06:41:35,586 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 555e2b65f40e49a8b6fc1cd9a3c7e089, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731307293298 2024-11-11T06:41:35,586 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting b918d529df9f449fa954cccdd2bfc716, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731307293298 2024-11-11T06:41:35,586 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13eab3af82be494db370fedeb4b7ab1c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307293951 2024-11-11T06:41:35,587 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 28d373bba576471b8f07b6bb95d95010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307293951 2024-11-11T06:41:35,598 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:35,603 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111469a7da8b4e947b896935312b32b917c_1a604f2419334e3a9e7d9b217f70c4d8 store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:35,605 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:35,605 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/0de9209d129e41d097db54e0ebc431cd is 50, key is test_row_0/B:col10/1731307293952/Put/seqid=0 2024-11-11T06:41:35,606 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111469a7da8b4e947b896935312b32b917c_1a604f2419334e3a9e7d9b217f70c4d8, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:35,606 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111469a7da8b4e947b896935312b32b917c_1a604f2419334e3a9e7d9b217f70c4d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:35,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742159_1335 (size=12104) 2024-11-11T06:41:35,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742158_1334 (size=4469) 2024-11-11T06:41:35,620 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#A#compaction#282 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:35,621 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/7b320406bd454cc9ae5ef01e144886cb is 175, key is test_row_0/A:col10/1731307293952/Put/seqid=0 2024-11-11T06:41:35,624 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/0de9209d129e41d097db54e0ebc431cd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/0de9209d129e41d097db54e0ebc431cd 2024-11-11T06:41:35,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742160_1336 (size=31058) 2024-11-11T06:41:35,635 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/B of 1a604f2419334e3a9e7d9b217f70c4d8 into 0de9209d129e41d097db54e0ebc431cd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:35,636 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:35,636 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/B, priority=13, startTime=1731307295582; duration=0sec 2024-11-11T06:41:35,636 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:35,636 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:B 2024-11-11T06:41:35,636 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:35,637 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:35,637 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/C is initiating minor compaction (all files) 2024-11-11T06:41:35,637 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/C in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:35,637 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/59bdc12f30954bab8f5fbd3f5c602c88, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/170ec40573de49f396bed16936260ae7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/04a6aa8212f54acc9cc73e1475797f27] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.2 K 2024-11-11T06:41:35,638 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 59bdc12f30954bab8f5fbd3f5c602c88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731307293246 2024-11-11T06:41:35,638 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/7b320406bd454cc9ae5ef01e144886cb as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7b320406bd454cc9ae5ef01e144886cb 2024-11-11T06:41:35,639 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 170ec40573de49f396bed16936260ae7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731307293298 2024-11-11T06:41:35,640 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 04a6aa8212f54acc9cc73e1475797f27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307293951 2024-11-11T06:41:35,648 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:35,648 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/19c1cb35e39042eab0e0aca7378791cf is 50, key is test_row_0/C:col10/1731307293952/Put/seqid=0 2024-11-11T06:41:35,650 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/A of 1a604f2419334e3a9e7d9b217f70c4d8 into 7b320406bd454cc9ae5ef01e144886cb(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:35,650 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:35,650 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/A, priority=13, startTime=1731307295582; duration=0sec 2024-11-11T06:41:35,650 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:35,650 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:A 2024-11-11T06:41:35,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742161_1337 (size=12104) 2024-11-11T06:41:35,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T06:41:35,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:35,699 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T06:41:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117218b25ad9d347f7ad318ece1659121d_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307295170/Put/seqid=0 2024-11-11T06:41:35,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742162_1338 (size=12154) 2024-11-11T06:41:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T06:41:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:35,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:35,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307355830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307355831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307355834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307355835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307355939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307355949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307355949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307355949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:35,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T06:41:36,058 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/19c1cb35e39042eab0e0aca7378791cf as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19c1cb35e39042eab0e0aca7378791cf 2024-11-11T06:41:36,062 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/C of 1a604f2419334e3a9e7d9b217f70c4d8 into 19c1cb35e39042eab0e0aca7378791cf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:36,062 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:36,062 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/C, priority=13, startTime=1731307295583; duration=0sec 2024-11-11T06:41:36,062 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:36,062 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:C 2024-11-11T06:41:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:36,113 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117218b25ad9d347f7ad318ece1659121d_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117218b25ad9d347f7ad318ece1659121d_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:36,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/690d86a056814aa79ff6fc8550b52251, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:36,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/690d86a056814aa79ff6fc8550b52251 is 175, key is test_row_0/A:col10/1731307295170/Put/seqid=0 2024-11-11T06:41:36,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742163_1339 (size=30955) 2024-11-11T06:41:36,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307356149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307356159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307356159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307356168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307356459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307356469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:36,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307356469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307356479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:36,518 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/690d86a056814aa79ff6fc8550b52251 2024-11-11T06:41:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/8cfa60e7911c4e4f8b3354d398933788 is 50, key is test_row_0/B:col10/1731307295170/Put/seqid=0 2024-11-11T06:41:36,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742164_1340 (size=12001) 2024-11-11T06:41:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T06:41:36,928 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/8cfa60e7911c4e4f8b3354d398933788 2024-11-11T06:41:36,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/ce664a231aec4445a7b8d5aae00b4df0 is 50, key is test_row_0/C:col10/1731307295170/Put/seqid=0 2024-11-11T06:41:36,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742165_1341 (size=12001) 2024-11-11T06:41:36,947 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/ce664a231aec4445a7b8d5aae00b4df0 2024-11-11T06:41:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/690d86a056814aa79ff6fc8550b52251 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/690d86a056814aa79ff6fc8550b52251 2024-11-11T06:41:36,958 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/690d86a056814aa79ff6fc8550b52251, entries=150, sequenceid=78, filesize=30.2 K 2024-11-11T06:41:36,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/8cfa60e7911c4e4f8b3354d398933788 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/8cfa60e7911c4e4f8b3354d398933788 2024-11-11T06:41:36,963 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/8cfa60e7911c4e4f8b3354d398933788, entries=150, sequenceid=78, filesize=11.7 K 2024-11-11T06:41:36,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/ce664a231aec4445a7b8d5aae00b4df0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/ce664a231aec4445a7b8d5aae00b4df0 2024-11-11T06:41:36,970 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/ce664a231aec4445a7b8d5aae00b4df0, entries=150, sequenceid=78, filesize=11.7 K 2024-11-11T06:41:36,971 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1272ms, sequenceid=78, compaction requested=false 2024-11-11T06:41:36,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:36,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:36,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T06:41:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T06:41:36,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T06:41:36,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5810 sec 2024-11-11T06:41:36,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5850 sec 2024-11-11T06:41:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:36,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:36,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:41:36,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115a0c194dd1724e7eb382cc6f2a052614_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307295828/Put/seqid=0 2024-11-11T06:41:36,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742166_1342 (size=14594) 2024-11-11T06:41:37,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307357023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307357027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307357034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307357034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307357139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307357139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307357149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307357149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307357349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307357349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307357359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307357359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:37,394 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115a0c194dd1724e7eb382cc6f2a052614_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a0c194dd1724e7eb382cc6f2a052614_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:37,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c6e01e95315542948a03478e0825b175, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:37,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c6e01e95315542948a03478e0825b175 is 175, key is test_row_0/A:col10/1731307295828/Put/seqid=0 2024-11-11T06:41:37,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742167_1343 (size=39549) 2024-11-11T06:41:37,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307357499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,507 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1a604f2419334e3a9e7d9b217f70c4d8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T06:41:37,539 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:37,539 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T06:41:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T06:41:37,540 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:37,541 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:37,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T06:41:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307357659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307357659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307357669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307357669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:37,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T06:41:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:37,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:37,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c6e01e95315542948a03478e0825b175 2024-11-11T06:41:37,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/24ed63535f254a12993bcfbbdd67f140 is 50, key is test_row_0/B:col10/1731307295828/Put/seqid=0 2024-11-11T06:41:37,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742168_1344 (size=12001) 2024-11-11T06:41:37,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T06:41:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:37,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:37,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T06:41:37,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T06:41:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:37,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T06:41:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T06:41:38,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307358169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:38,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307358169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:38,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307358178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:38,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307358189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:38,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:38,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:38,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:38,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/24ed63535f254a12993bcfbbdd67f140 2024-11-11T06:41:38,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/98ec7e64b55841449932b88449ba3fb4 is 50, key is test_row_0/C:col10/1731307295828/Put/seqid=0 2024-11-11T06:41:38,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742169_1345 (size=12001) 2024-11-11T06:41:38,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T06:41:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T06:41:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T06:41:38,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:38,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/98ec7e64b55841449932b88449ba3fb4 2024-11-11T06:41:38,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c6e01e95315542948a03478e0825b175 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c6e01e95315542948a03478e0825b175 2024-11-11T06:41:38,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c6e01e95315542948a03478e0825b175, entries=200, sequenceid=92, filesize=38.6 K 2024-11-11T06:41:38,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/24ed63535f254a12993bcfbbdd67f140 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24ed63535f254a12993bcfbbdd67f140 2024-11-11T06:41:38,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24ed63535f254a12993bcfbbdd67f140, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T06:41:38,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/98ec7e64b55841449932b88449ba3fb4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/98ec7e64b55841449932b88449ba3fb4 2024-11-11T06:41:38,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/98ec7e64b55841449932b88449ba3fb4, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T06:41:38,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1665ms, sequenceid=92, compaction requested=true 2024-11-11T06:41:38,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:38,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:38,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:38,643 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:38,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:38,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:38,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:38,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T06:41:38,643 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:38,644 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:38,644 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/A is initiating minor compaction (all files) 2024-11-11T06:41:38,644 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/A in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,644 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7b320406bd454cc9ae5ef01e144886cb, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/690d86a056814aa79ff6fc8550b52251, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c6e01e95315542948a03478e0825b175] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=99.2 K 2024-11-11T06:41:38,644 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,645 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7b320406bd454cc9ae5ef01e144886cb, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/690d86a056814aa79ff6fc8550b52251, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c6e01e95315542948a03478e0825b175] 2024-11-11T06:41:38,645 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:38,645 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b320406bd454cc9ae5ef01e144886cb, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307293951 2024-11-11T06:41:38,645 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/B is initiating minor compaction (all files) 2024-11-11T06:41:38,645 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/B in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,645 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/0de9209d129e41d097db54e0ebc431cd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/8cfa60e7911c4e4f8b3354d398933788, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24ed63535f254a12993bcfbbdd67f140] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.3 K 2024-11-11T06:41:38,645 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 690d86a056814aa79ff6fc8550b52251, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731307295155 2024-11-11T06:41:38,645 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de9209d129e41d097db54e0ebc431cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307293951 2024-11-11T06:41:38,645 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6e01e95315542948a03478e0825b175, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307295828 2024-11-11T06:41:38,646 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cfa60e7911c4e4f8b3354d398933788, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731307295155 2024-11-11T06:41:38,646 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ed63535f254a12993bcfbbdd67f140, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307295828 2024-11-11T06:41:38,651 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:38,652 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/f2a07ad17c0d4d568bded44e4ba83d30 is 50, key is test_row_0/B:col10/1731307295828/Put/seqid=0 2024-11-11T06:41:38,652 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:38,663 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112475a2e575d14a8ea78783c00ed065ad_1a604f2419334e3a9e7d9b217f70c4d8 store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:38,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742170_1346 (size=12207) 2024-11-11T06:41:38,665 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112475a2e575d14a8ea78783c00ed065ad_1a604f2419334e3a9e7d9b217f70c4d8, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:38,665 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112475a2e575d14a8ea78783c00ed065ad_1a604f2419334e3a9e7d9b217f70c4d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:38,669 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/f2a07ad17c0d4d568bded44e4ba83d30 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/f2a07ad17c0d4d568bded44e4ba83d30 2024-11-11T06:41:38,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742171_1347 (size=4469) 2024-11-11T06:41:38,674 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#A#compaction#292 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:38,675 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/330cbc4d987649a98899178e42142aea is 175, key is test_row_0/A:col10/1731307295828/Put/seqid=0 2024-11-11T06:41:38,676 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/B of 1a604f2419334e3a9e7d9b217f70c4d8 into f2a07ad17c0d4d568bded44e4ba83d30(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:38,676 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:38,676 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/B, priority=13, startTime=1731307298643; duration=0sec 2024-11-11T06:41:38,676 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:38,676 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:B 2024-11-11T06:41:38,676 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:38,678 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:38,678 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/C is initiating minor compaction (all files) 2024-11-11T06:41:38,678 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/C in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,678 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19c1cb35e39042eab0e0aca7378791cf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/ce664a231aec4445a7b8d5aae00b4df0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/98ec7e64b55841449932b88449ba3fb4] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.3 K 2024-11-11T06:41:38,678 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c1cb35e39042eab0e0aca7378791cf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307293951 2024-11-11T06:41:38,679 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ce664a231aec4445a7b8d5aae00b4df0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731307295155 2024-11-11T06:41:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T06:41:38,679 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ec7e64b55841449932b88449ba3fb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307295828 2024-11-11T06:41:38,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742172_1348 (size=31161) 2024-11-11T06:41:38,687 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:38,687 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/2d15c5239654424182063de27fc6d7a7 is 50, key is test_row_0/C:col10/1731307295828/Put/seqid=0 2024-11-11T06:41:38,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742173_1349 (size=12207) 2024-11-11T06:41:38,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T06:41:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:38,761 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T06:41:38,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114bd78bb7a3a24a469fd88b0bd2e2f353_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307297033/Put/seqid=0 2024-11-11T06:41:38,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742174_1350 (size=12154) 2024-11-11T06:41:39,084 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/330cbc4d987649a98899178e42142aea as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/330cbc4d987649a98899178e42142aea 2024-11-11T06:41:39,088 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/A of 1a604f2419334e3a9e7d9b217f70c4d8 into 330cbc4d987649a98899178e42142aea(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:39,088 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:39,088 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/A, priority=13, startTime=1731307298643; duration=0sec 2024-11-11T06:41:39,088 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:39,088 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:A 2024-11-11T06:41:39,100 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/2d15c5239654424182063de27fc6d7a7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2d15c5239654424182063de27fc6d7a7 2024-11-11T06:41:39,103 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/C of 1a604f2419334e3a9e7d9b217f70c4d8 into 2d15c5239654424182063de27fc6d7a7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:39,103 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:39,103 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/C, priority=13, startTime=1731307298643; duration=0sec 2024-11-11T06:41:39,103 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:39,103 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:C 2024-11-11T06:41:39,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:39,175 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114bd78bb7a3a24a469fd88b0bd2e2f353_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114bd78bb7a3a24a469fd88b0bd2e2f353_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/7df6c7cfc33d40cd8e878cb5ec168abd, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/7df6c7cfc33d40cd8e878cb5ec168abd is 175, key is test_row_0/A:col10/1731307297033/Put/seqid=0 2024-11-11T06:41:39,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742175_1351 (size=30955) 2024-11-11T06:41:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:39,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307359209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,220 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in 1a604f2419334e3a9e7d9b217f70c4d8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307359218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307359218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307359220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307359339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307359339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307359339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307359549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307359549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307359549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,580 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/7df6c7cfc33d40cd8e878cb5ec168abd 2024-11-11T06:41:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/691c8f17b0294d9c94ecfd793b93374c is 50, key is test_row_0/B:col10/1731307297033/Put/seqid=0 2024-11-11T06:41:39,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742176_1352 (size=12001) 2024-11-11T06:41:39,599 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/691c8f17b0294d9c94ecfd793b93374c 2024-11-11T06:41:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/0b9a2bf649b7478e989eb8a3a75b1263 is 50, key is test_row_0/C:col10/1731307297033/Put/seqid=0 2024-11-11T06:41:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742177_1353 (size=12001) 2024-11-11T06:41:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T06:41:39,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307359869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307359869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307359870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,012 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/0b9a2bf649b7478e989eb8a3a75b1263 2024-11-11T06:41:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/7df6c7cfc33d40cd8e878cb5ec168abd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7df6c7cfc33d40cd8e878cb5ec168abd 2024-11-11T06:41:40,019 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7df6c7cfc33d40cd8e878cb5ec168abd, entries=150, sequenceid=116, filesize=30.2 K 2024-11-11T06:41:40,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/691c8f17b0294d9c94ecfd793b93374c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/691c8f17b0294d9c94ecfd793b93374c 2024-11-11T06:41:40,022 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/691c8f17b0294d9c94ecfd793b93374c, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T06:41:40,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/0b9a2bf649b7478e989eb8a3a75b1263 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/0b9a2bf649b7478e989eb8a3a75b1263 2024-11-11T06:41:40,026 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/0b9a2bf649b7478e989eb8a3a75b1263, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T06:41:40,028 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1267ms, sequenceid=116, compaction requested=false 2024-11-11T06:41:40,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:40,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:40,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T06:41:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T06:41:40,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T06:41:40,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4880 sec 2024-11-11T06:41:40,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.4920 sec 2024-11-11T06:41:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:40,384 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:40,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T06:41:40,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e8ad27022e1b4488b60a10b4a38262e3_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307299193/Put/seqid=0 2024-11-11T06:41:40,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742178_1354 (size=14694) 2024-11-11T06:41:40,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307360456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307360465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307360466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307360579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307360579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307360579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307360789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:40,799 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e8ad27022e1b4488b60a10b4a38262e3_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e8ad27022e1b4488b60a10b4a38262e3_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:40,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/774d3917b0e64f6e92ec6110c1016512, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:40,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/774d3917b0e64f6e92ec6110c1016512 is 175, key is test_row_0/A:col10/1731307299193/Put/seqid=0 2024-11-11T06:41:40,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742179_1355 (size=39649) 2024-11-11T06:41:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307360799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307360799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307361098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307361109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:41,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307361109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:41,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,205 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/774d3917b0e64f6e92ec6110c1016512 2024-11-11T06:41:41,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/c6bd93891c7d4503981a9647920faaff is 50, key is test_row_0/B:col10/1731307299193/Put/seqid=0 2024-11-11T06:41:41,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742180_1356 (size=12101) 2024-11-11T06:41:41,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/c6bd93891c7d4503981a9647920faaff 2024-11-11T06:41:41,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/bc875f3598bb4f9f8011600132807710 is 50, key is test_row_0/C:col10/1731307299193/Put/seqid=0 2024-11-11T06:41:41,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742181_1357 (size=12101) 2024-11-11T06:41:41,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307361249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:41,260 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in 1a604f2419334e3a9e7d9b217f70c4d8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307361549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:41,556 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1a604f2419334e3a9e7d9b217f70c4d8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307361608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307361619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:41,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307361619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:41,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/bc875f3598bb4f9f8011600132807710 2024-11-11T06:41:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/774d3917b0e64f6e92ec6110c1016512 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/774d3917b0e64f6e92ec6110c1016512 2024-11-11T06:41:41,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/774d3917b0e64f6e92ec6110c1016512, entries=200, sequenceid=132, filesize=38.7 K 2024-11-11T06:41:41,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/c6bd93891c7d4503981a9647920faaff as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/c6bd93891c7d4503981a9647920faaff 2024-11-11T06:41:41,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/c6bd93891c7d4503981a9647920faaff, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T06:41:41,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/bc875f3598bb4f9f8011600132807710 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/bc875f3598bb4f9f8011600132807710 2024-11-11T06:41:41,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/bc875f3598bb4f9f8011600132807710, entries=150, sequenceid=132, filesize=11.8 K 2024-11-11T06:41:41,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1257ms, sequenceid=132, compaction requested=true 2024-11-11T06:41:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:41,642 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:41,642 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:41,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:41,642 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:41,643 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/B is initiating minor compaction (all files) 2024-11-11T06:41:41,643 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/B in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:41,643 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/f2a07ad17c0d4d568bded44e4ba83d30, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/691c8f17b0294d9c94ecfd793b93374c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/c6bd93891c7d4503981a9647920faaff] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.5 K 2024-11-11T06:41:41,643 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:41,643 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/A is initiating minor compaction (all files) 2024-11-11T06:41:41,643 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/A in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:41,643 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/330cbc4d987649a98899178e42142aea, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7df6c7cfc33d40cd8e878cb5ec168abd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/774d3917b0e64f6e92ec6110c1016512] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=99.4 K 2024-11-11T06:41:41,643 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a07ad17c0d4d568bded44e4ba83d30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307295828 2024-11-11T06:41:41,643 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:41,643 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/330cbc4d987649a98899178e42142aea, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7df6c7cfc33d40cd8e878cb5ec168abd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/774d3917b0e64f6e92ec6110c1016512] 2024-11-11T06:41:41,643 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 691c8f17b0294d9c94ecfd793b93374c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731307297020 2024-11-11T06:41:41,644 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 330cbc4d987649a98899178e42142aea, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307295828 2024-11-11T06:41:41,644 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c6bd93891c7d4503981a9647920faaff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307299193 2024-11-11T06:41:41,644 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7df6c7cfc33d40cd8e878cb5ec168abd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731307297020 2024-11-11T06:41:41,644 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774d3917b0e64f6e92ec6110c1016512, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307299193 2024-11-11T06:41:41,650 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:41,650 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/32449970c68742388e864686569d6d67 is 50, key is test_row_0/B:col10/1731307299193/Put/seqid=0 2024-11-11T06:41:41,651 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:41,653 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111afd2a9b29f11440dbb23ee68d2e23d53_1a604f2419334e3a9e7d9b217f70c4d8 store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:41,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742182_1358 (size=12409) 2024-11-11T06:41:41,656 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111afd2a9b29f11440dbb23ee68d2e23d53_1a604f2419334e3a9e7d9b217f70c4d8, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:41,656 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111afd2a9b29f11440dbb23ee68d2e23d53_1a604f2419334e3a9e7d9b217f70c4d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:41,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742183_1359 (size=4469) 2024-11-11T06:41:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T06:41:41,719 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:41,720 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T06:41:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T06:41:41,721 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:41,722 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:41,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T06:41:41,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T06:41:41,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:41,874 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T06:41:41,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fd999bdc56a74d5296473638e048c4ac_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307300463/Put/seqid=0 2024-11-11T06:41:41,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742184_1360 (size=12304) 2024-11-11T06:41:41,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T06:41:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T06:41:42,059 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/32449970c68742388e864686569d6d67 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/32449970c68742388e864686569d6d67 2024-11-11T06:41:42,063 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/B of 1a604f2419334e3a9e7d9b217f70c4d8 into 32449970c68742388e864686569d6d67(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:42,063 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:42,063 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/B, priority=13, startTime=1731307301642; duration=0sec 2024-11-11T06:41:42,063 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:42,063 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:B 2024-11-11T06:41:42,063 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:42,064 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:42,064 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/C is initiating minor compaction (all files) 2024-11-11T06:41:42,064 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/C in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:42,064 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2d15c5239654424182063de27fc6d7a7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/0b9a2bf649b7478e989eb8a3a75b1263, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/bc875f3598bb4f9f8011600132807710] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.5 K 2024-11-11T06:41:42,064 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:42,065 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d15c5239654424182063de27fc6d7a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731307295828 2024-11-11T06:41:42,065 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9a2bf649b7478e989eb8a3a75b1263, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731307297020 2024-11-11T06:41:42,065 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/3aa84bf139044e6a9e0a484d211cf85c is 175, key is test_row_0/A:col10/1731307299193/Put/seqid=0 2024-11-11T06:41:42,065 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting bc875f3598bb4f9f8011600132807710, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307299193 2024-11-11T06:41:42,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742185_1361 (size=31363) 2024-11-11T06:41:42,074 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#C#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:42,074 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/1d26248d4fa44d80b388b1ad9484b14a is 50, key is test_row_0/C:col10/1731307299193/Put/seqid=0 2024-11-11T06:41:42,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742186_1362 (size=12409) 2024-11-11T06:41:42,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:42,287 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fd999bdc56a74d5296473638e048c4ac_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fd999bdc56a74d5296473638e048c4ac_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:42,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:42,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd is 175, key is test_row_0/A:col10/1731307300463/Put/seqid=0 2024-11-11T06:41:42,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742187_1363 (size=31105) 2024-11-11T06:41:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T06:41:42,474 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/3aa84bf139044e6a9e0a484d211cf85c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3aa84bf139044e6a9e0a484d211cf85c 2024-11-11T06:41:42,478 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/A of 1a604f2419334e3a9e7d9b217f70c4d8 into 3aa84bf139044e6a9e0a484d211cf85c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:42,478 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:42,478 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/A, priority=13, startTime=1731307301642; duration=0sec 2024-11-11T06:41:42,478 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:42,478 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:A 2024-11-11T06:41:42,482 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/1d26248d4fa44d80b388b1ad9484b14a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/1d26248d4fa44d80b388b1ad9484b14a 2024-11-11T06:41:42,486 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/C of 1a604f2419334e3a9e7d9b217f70c4d8 into 1d26248d4fa44d80b388b1ad9484b14a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:42,486 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:42,486 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/C, priority=13, startTime=1731307301642; duration=0sec 2024-11-11T06:41:42,486 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:42,486 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:C 2024-11-11T06:41:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:42,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:42,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307362643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307362645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:42,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307362645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:42,692 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd 2024-11-11T06:41:42,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/e94d67b461c246cab26a068dc41183ac is 50, key is test_row_0/B:col10/1731307300463/Put/seqid=0 2024-11-11T06:41:42,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742188_1364 (size=12151) 2024-11-11T06:41:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307362759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307362759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307362759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T06:41:42,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307362969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307362969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:42,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307362969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,141 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/e94d67b461c246cab26a068dc41183ac 2024-11-11T06:41:43,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/311bdf92e3bb4d10923151c340789076 is 50, key is test_row_0/C:col10/1731307300463/Put/seqid=0 2024-11-11T06:41:43,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742189_1365 (size=12151) 2024-11-11T06:41:43,155 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/311bdf92e3bb4d10923151c340789076 2024-11-11T06:41:43,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd 2024-11-11T06:41:43,163 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd, entries=150, sequenceid=154, filesize=30.4 K 2024-11-11T06:41:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/e94d67b461c246cab26a068dc41183ac as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e94d67b461c246cab26a068dc41183ac 2024-11-11T06:41:43,167 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e94d67b461c246cab26a068dc41183ac, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T06:41:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/311bdf92e3bb4d10923151c340789076 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/311bdf92e3bb4d10923151c340789076 2024-11-11T06:41:43,170 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/311bdf92e3bb4d10923151c340789076, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T06:41:43,171 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1297ms, sequenceid=154, compaction requested=false 2024-11-11T06:41:43,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:43,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:43,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T06:41:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T06:41:43,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T06:41:43,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4500 sec 2024-11-11T06:41:43,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4540 sec 2024-11-11T06:41:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:43,284 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:43,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T06:41:43,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111111eb29ee2d0d4d63bc0fca60534ee8c7_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307302644/Put/seqid=0 2024-11-11T06:41:43,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742190_1366 (size=14794) 2024-11-11T06:41:43,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307363324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307363325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307363328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307363439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307363439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307363439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307363649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307363649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307363649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:43,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111111eb29ee2d0d4d63bc0fca60534ee8c7_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111111eb29ee2d0d4d63bc0fca60534ee8c7_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:43,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c768e58a3b3a43b9ae1b631a501ead8e, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:43,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c768e58a3b3a43b9ae1b631a501ead8e is 175, key is test_row_0/A:col10/1731307302644/Put/seqid=0 2024-11-11T06:41:43,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742191_1367 (size=39749) 2024-11-11T06:41:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T06:41:43,879 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:43,880 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-11T06:41:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T06:41:43,881 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:43,882 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:43,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:43,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307363959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307363959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307363969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T06:41:44,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T06:41:44,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:44,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,107 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c768e58a3b3a43b9ae1b631a501ead8e 2024-11-11T06:41:44,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/5ae0017633704c33a0c0afbae3cc5f26 is 50, key is test_row_0/B:col10/1731307302644/Put/seqid=0 2024-11-11T06:41:44,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742192_1368 (size=12151) 2024-11-11T06:41:44,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T06:41:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T06:41:44,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T06:41:44,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:44,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307364469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:44,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307364479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:44,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T06:41:44,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:44,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:44,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307364489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T06:41:44,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/5ae0017633704c33a0c0afbae3cc5f26 2024-11-11T06:41:44,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/f9543ce2d6174987a3c2cda3ba4ad88e is 50, key is test_row_0/C:col10/1731307302644/Put/seqid=0 2024-11-11T06:41:44,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742193_1369 (size=12151) 2024-11-11T06:41:44,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/f9543ce2d6174987a3c2cda3ba4ad88e 2024-11-11T06:41:44,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c768e58a3b3a43b9ae1b631a501ead8e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c768e58a3b3a43b9ae1b631a501ead8e 2024-11-11T06:41:44,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c768e58a3b3a43b9ae1b631a501ead8e, entries=200, sequenceid=172, filesize=38.8 K 2024-11-11T06:41:44,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/5ae0017633704c33a0c0afbae3cc5f26 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ae0017633704c33a0c0afbae3cc5f26 2024-11-11T06:41:44,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ae0017633704c33a0c0afbae3cc5f26, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T06:41:44,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/f9543ce2d6174987a3c2cda3ba4ad88e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/f9543ce2d6174987a3c2cda3ba4ad88e 2024-11-11T06:41:44,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/f9543ce2d6174987a3c2cda3ba4ad88e, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T06:41:44,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1261ms, sequenceid=172, compaction requested=true 2024-11-11T06:41:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:44,545 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:44,545 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:44,546 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:44,546 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/A is initiating minor compaction (all files) 2024-11-11T06:41:44,546 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/A in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,547 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3aa84bf139044e6a9e0a484d211cf85c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c768e58a3b3a43b9ae1b631a501ead8e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=99.8 K 2024-11-11T06:41:44,547 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,547 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3aa84bf139044e6a9e0a484d211cf85c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c768e58a3b3a43b9ae1b631a501ead8e] 2024-11-11T06:41:44,547 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:44,547 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/B is initiating minor compaction (all files) 2024-11-11T06:41:44,547 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/B in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,547 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa84bf139044e6a9e0a484d211cf85c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307299193 2024-11-11T06:41:44,547 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/32449970c68742388e864686569d6d67, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e94d67b461c246cab26a068dc41183ac, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ae0017633704c33a0c0afbae3cc5f26] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.9 K 2024-11-11T06:41:44,547 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8fbc28d606a4c0e9e0d8c41cbc07ecd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731307300453 2024-11-11T06:41:44,547 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 32449970c68742388e864686569d6d67, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307299193 2024-11-11T06:41:44,547 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c768e58a3b3a43b9ae1b631a501ead8e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307302636 2024-11-11T06:41:44,547 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting e94d67b461c246cab26a068dc41183ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731307300453 2024-11-11T06:41:44,548 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae0017633704c33a0c0afbae3cc5f26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307302643 2024-11-11T06:41:44,556 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:44,556 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/e68a71ffb50f436486442c908fc65fb4 is 50, key is test_row_0/B:col10/1731307302644/Put/seqid=0 2024-11-11T06:41:44,557 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:44,559 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411111d31934de0b646958aa325868faf03f1_1a604f2419334e3a9e7d9b217f70c4d8 store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:44,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742194_1370 (size=12561) 2024-11-11T06:41:44,561 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411111d31934de0b646958aa325868faf03f1_1a604f2419334e3a9e7d9b217f70c4d8, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:44,561 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111d31934de0b646958aa325868faf03f1_1a604f2419334e3a9e7d9b217f70c4d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:44,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742195_1371 (size=4469) 2024-11-11T06:41:44,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T06:41:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,644 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T06:41:44,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116532dbfe105d481ab1cc9cd118be3a76_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307303327/Put/seqid=0 2024-11-11T06:41:44,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742196_1372 (size=12304) 2024-11-11T06:41:44,964 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/e68a71ffb50f436486442c908fc65fb4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e68a71ffb50f436486442c908fc65fb4 2024-11-11T06:41:44,968 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:44,968 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/9e65ee9838414cb28e45ca008a9db4c2 is 175, key is test_row_0/A:col10/1731307302644/Put/seqid=0 2024-11-11T06:41:44,969 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/B of 1a604f2419334e3a9e7d9b217f70c4d8 into e68a71ffb50f436486442c908fc65fb4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:44,969 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:44,969 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/B, priority=13, startTime=1731307304545; duration=0sec 2024-11-11T06:41:44,969 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:44,969 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:B 2024-11-11T06:41:44,969 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:44,970 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:44,970 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/C is initiating minor compaction (all files) 2024-11-11T06:41:44,970 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/C in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:44,970 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/1d26248d4fa44d80b388b1ad9484b14a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/311bdf92e3bb4d10923151c340789076, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/f9543ce2d6174987a3c2cda3ba4ad88e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.9 K 2024-11-11T06:41:44,970 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d26248d4fa44d80b388b1ad9484b14a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731307299193 2024-11-11T06:41:44,970 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 311bdf92e3bb4d10923151c340789076, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731307300453 2024-11-11T06:41:44,971 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting f9543ce2d6174987a3c2cda3ba4ad88e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307302643 2024-11-11T06:41:44,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742197_1373 (size=31515) 2024-11-11T06:41:44,978 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:44,978 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/699e9a2b9bcb426c9174765f4962edcf is 50, key is test_row_0/C:col10/1731307302644/Put/seqid=0 2024-11-11T06:41:44,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742198_1374 (size=12561) 2024-11-11T06:41:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T06:41:45,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,058 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116532dbfe105d481ab1cc9cd118be3a76_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116532dbfe105d481ab1cc9cd118be3a76_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/ed00493fb600494484e4d24053c57f0b, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/ed00493fb600494484e4d24053c57f0b is 175, key is test_row_0/A:col10/1731307303327/Put/seqid=0 2024-11-11T06:41:45,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742199_1375 (size=31105) 2024-11-11T06:41:45,064 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/ed00493fb600494484e4d24053c57f0b 2024-11-11T06:41:45,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/058f887b2694428c8ca6110f126ef87b is 50, key is test_row_0/B:col10/1731307303327/Put/seqid=0 2024-11-11T06:41:45,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742200_1376 (size=12151) 2024-11-11T06:41:45,088 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/058f887b2694428c8ca6110f126ef87b 2024-11-11T06:41:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/19a4c179014c4c538525ef43703b0e79 is 50, key is test_row_0/C:col10/1731307303327/Put/seqid=0 2024-11-11T06:41:45,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742201_1377 (size=12151) 2024-11-11T06:41:45,118 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/19a4c179014c4c538525ef43703b0e79 2024-11-11T06:41:45,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/ed00493fb600494484e4d24053c57f0b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/ed00493fb600494484e4d24053c57f0b 2024-11-11T06:41:45,127 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/ed00493fb600494484e4d24053c57f0b, entries=150, sequenceid=193, filesize=30.4 K 2024-11-11T06:41:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/058f887b2694428c8ca6110f126ef87b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/058f887b2694428c8ca6110f126ef87b 2024-11-11T06:41:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,133 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/058f887b2694428c8ca6110f126ef87b, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T06:41:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/19a4c179014c4c538525ef43703b0e79 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19a4c179014c4c538525ef43703b0e79 2024-11-11T06:41:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,141 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19a4c179014c4c538525ef43703b0e79, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T06:41:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 1a604f2419334e3a9e7d9b217f70c4d8 in 497ms, sequenceid=193, compaction requested=false 2024-11-11T06:41:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-11T06:41:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-11T06:41:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T06:41:45,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2610 sec 2024-11-11T06:41:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.2660 sec 2024-11-11T06:41:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,380 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/9e65ee9838414cb28e45ca008a9db4c2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9e65ee9838414cb28e45ca008a9db4c2 2024-11-11T06:41:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,385 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/A of 1a604f2419334e3a9e7d9b217f70c4d8 into 9e65ee9838414cb28e45ca008a9db4c2(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:45,385 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:45,385 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/A, priority=13, startTime=1731307304545; duration=0sec 2024-11-11T06:41:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,385 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:45,385 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:A 2024-11-11T06:41:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,387 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/699e9a2b9bcb426c9174765f4962edcf as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/699e9a2b9bcb426c9174765f4962edcf 2024-11-11T06:41:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,391 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/C of 1a604f2419334e3a9e7d9b217f70c4d8 into 699e9a2b9bcb426c9174765f4962edcf(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:45,391 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:45,391 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/C, priority=13, startTime=1731307304545; duration=0sec 2024-11-11T06:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,392 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:45,392 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:C 2024-11-11T06:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:45,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:45,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:41:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113ecb96a05f66434aa9f2c35225c4b109_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307305500/Put/seqid=0 2024-11-11T06:41:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742202_1378 (size=22268) 2024-11-11T06:41:45,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307365598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307365598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307365608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307365609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307365719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307365719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307365720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307365729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307365929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,941 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:45,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307365939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307365939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,944 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113ecb96a05f66434aa9f2c35225c4b109_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113ecb96a05f66434aa9f2c35225c4b109_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:45,945 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/5fa5ed39ce2a4df1a7642d077e961e1a, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:45,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/5fa5ed39ce2a4df1a7642d077e961e1a is 175, key is test_row_0/A:col10/1731307305500/Put/seqid=0 2024-11-11T06:41:45,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742203_1379 (size=65673) 2024-11-11T06:41:45,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/5fa5ed39ce2a4df1a7642d077e961e1a 2024-11-11T06:41:45,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307365949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:45,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/2981c6951df24b5cb0cff6fd2c9dab40 is 50, key is test_row_0/B:col10/1731307305500/Put/seqid=0 2024-11-11T06:41:45,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742204_1380 (size=12151) 2024-11-11T06:41:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T06:41:46,029 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:46,030 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-11T06:41:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T06:41:46,031 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:46,033 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:46,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T06:41:46,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T06:41:46,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:46,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307366249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:46,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307366249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:46,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:46,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307366249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:46,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:46,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307366259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:46,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T06:41:46,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T06:41:46,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/2981c6951df24b5cb0cff6fd2c9dab40 2024-11-11T06:41:46,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/2bf4986361a04e06a5294b67fd80cf32 is 50, key is test_row_0/C:col10/1731307305500/Put/seqid=0 2024-11-11T06:41:46,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742205_1381 (size=12151) 2024-11-11T06:41:46,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T06:41:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T06:41:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T06:41:46,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307366759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:46,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307366759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307366759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:46,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/2bf4986361a04e06a5294b67fd80cf32 2024-11-11T06:41:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307366769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:46,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/5fa5ed39ce2a4df1a7642d077e961e1a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/5fa5ed39ce2a4df1a7642d077e961e1a 2024-11-11T06:41:46,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/5fa5ed39ce2a4df1a7642d077e961e1a, entries=350, sequenceid=207, filesize=64.1 K 2024-11-11T06:41:46,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/2981c6951df24b5cb0cff6fd2c9dab40 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2981c6951df24b5cb0cff6fd2c9dab40 2024-11-11T06:41:46,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2981c6951df24b5cb0cff6fd2c9dab40, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T06:41:46,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/2bf4986361a04e06a5294b67fd80cf32 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2bf4986361a04e06a5294b67fd80cf32 2024-11-11T06:41:46,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2bf4986361a04e06a5294b67fd80cf32, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T06:41:46,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1279ms, sequenceid=207, compaction requested=true 2024-11-11T06:41:46,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:46,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:46,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:46,792 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:46,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:46,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:46,792 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:46,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:46,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:46,793 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:46,793 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:46,793 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/B is initiating minor compaction (all files) 2024-11-11T06:41:46,793 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/A is initiating minor compaction (all files) 2024-11-11T06:41:46,793 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/B in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,793 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/A in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,793 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e68a71ffb50f436486442c908fc65fb4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/058f887b2694428c8ca6110f126ef87b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2981c6951df24b5cb0cff6fd2c9dab40] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=36.0 K 2024-11-11T06:41:46,793 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9e65ee9838414cb28e45ca008a9db4c2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/ed00493fb600494484e4d24053c57f0b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/5fa5ed39ce2a4df1a7642d077e961e1a] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=125.3 K 2024-11-11T06:41:46,794 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,794 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9e65ee9838414cb28e45ca008a9db4c2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/ed00493fb600494484e4d24053c57f0b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/5fa5ed39ce2a4df1a7642d077e961e1a] 2024-11-11T06:41:46,794 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting e68a71ffb50f436486442c908fc65fb4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307302643 2024-11-11T06:41:46,794 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e65ee9838414cb28e45ca008a9db4c2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307302643 2024-11-11T06:41:46,794 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed00493fb600494484e4d24053c57f0b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731307303316 2024-11-11T06:41:46,794 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 058f887b2694428c8ca6110f126ef87b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731307303316 2024-11-11T06:41:46,795 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fa5ed39ce2a4df1a7642d077e961e1a, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307305436 2024-11-11T06:41:46,795 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2981c6951df24b5cb0cff6fd2c9dab40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307305500 2024-11-11T06:41:46,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T06:41:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:46,798 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T06:41:46,800 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:46,802 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-11T06:41:46,803 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/50515f2bedee4f7789b370eb59e9e8d2 is 50, key is test_row_0/B:col10/1731307305500/Put/seqid=0 2024-11-11T06:41:46,806 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ab340be838f9448483bbb67bd58bc5fd_1a604f2419334e3a9e7d9b217f70c4d8 store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:46,808 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ab340be838f9448483bbb67bd58bc5fd_1a604f2419334e3a9e7d9b217f70c4d8, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:46,808 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ab340be838f9448483bbb67bd58bc5fd_1a604f2419334e3a9e7d9b217f70c4d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:46,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742206_1382 (size=12663) 2024-11-11T06:41:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115b86b17023ee4259afd2d63fd1d592ff_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307305606/Put/seqid=0 2024-11-11T06:41:46,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742207_1383 (size=4469) 2024-11-11T06:41:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742208_1384 (size=12304) 2024-11-11T06:41:46,889 DEBUG [regionserver/ce2cf0244921:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-11T06:41:46,891 WARN [regionserver/ce2cf0244921:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-11T06:41:46,892 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/aea5e1d287b242bc820e41e4e58c3a26, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/b918d529df9f449fa954cccdd2bfc716, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7b320406bd454cc9ae5ef01e144886cb, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/13eab3af82be494db370fedeb4b7ab1c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/690d86a056814aa79ff6fc8550b52251, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c6e01e95315542948a03478e0825b175, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/330cbc4d987649a98899178e42142aea, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7df6c7cfc33d40cd8e878cb5ec168abd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/774d3917b0e64f6e92ec6110c1016512, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3aa84bf139044e6a9e0a484d211cf85c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c768e58a3b3a43b9ae1b631a501ead8e] to archive 2024-11-11T06:41:46,893 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:41:46,895 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/aea5e1d287b242bc820e41e4e58c3a26 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/aea5e1d287b242bc820e41e4e58c3a26 2024-11-11T06:41:46,896 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/b918d529df9f449fa954cccdd2bfc716 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/b918d529df9f449fa954cccdd2bfc716 2024-11-11T06:41:46,896 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7b320406bd454cc9ae5ef01e144886cb to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7b320406bd454cc9ae5ef01e144886cb 2024-11-11T06:41:46,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/13eab3af82be494db370fedeb4b7ab1c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/13eab3af82be494db370fedeb4b7ab1c 2024-11-11T06:41:46,900 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/690d86a056814aa79ff6fc8550b52251 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/690d86a056814aa79ff6fc8550b52251 2024-11-11T06:41:46,901 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c6e01e95315542948a03478e0825b175 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c6e01e95315542948a03478e0825b175 2024-11-11T06:41:46,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/330cbc4d987649a98899178e42142aea to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/330cbc4d987649a98899178e42142aea 2024-11-11T06:41:46,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7df6c7cfc33d40cd8e878cb5ec168abd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/7df6c7cfc33d40cd8e878cb5ec168abd 2024-11-11T06:41:46,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/774d3917b0e64f6e92ec6110c1016512 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/774d3917b0e64f6e92ec6110c1016512 2024-11-11T06:41:46,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3aa84bf139044e6a9e0a484d211cf85c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3aa84bf139044e6a9e0a484d211cf85c 2024-11-11T06:41:46,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d8fbc28d606a4c0e9e0d8c41cbc07ecd 2024-11-11T06:41:46,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c768e58a3b3a43b9ae1b631a501ead8e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c768e58a3b3a43b9ae1b631a501ead8e 2024-11-11T06:41:46,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/748acd342e2f4e5ea4eb015db37535bc, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/555e2b65f40e49a8b6fc1cd9a3c7e089, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/0de9209d129e41d097db54e0ebc431cd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/28d373bba576471b8f07b6bb95d95010, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/8cfa60e7911c4e4f8b3354d398933788, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/f2a07ad17c0d4d568bded44e4ba83d30, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24ed63535f254a12993bcfbbdd67f140, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/691c8f17b0294d9c94ecfd793b93374c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/32449970c68742388e864686569d6d67, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/c6bd93891c7d4503981a9647920faaff, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e94d67b461c246cab26a068dc41183ac, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ae0017633704c33a0c0afbae3cc5f26] to archive 2024-11-11T06:41:46,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:41:46,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/748acd342e2f4e5ea4eb015db37535bc to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/748acd342e2f4e5ea4eb015db37535bc 2024-11-11T06:41:46,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/555e2b65f40e49a8b6fc1cd9a3c7e089 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/555e2b65f40e49a8b6fc1cd9a3c7e089 2024-11-11T06:41:46,911 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/0de9209d129e41d097db54e0ebc431cd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/0de9209d129e41d097db54e0ebc431cd 2024-11-11T06:41:46,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/28d373bba576471b8f07b6bb95d95010 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/28d373bba576471b8f07b6bb95d95010 2024-11-11T06:41:46,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/8cfa60e7911c4e4f8b3354d398933788 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/8cfa60e7911c4e4f8b3354d398933788 2024-11-11T06:41:46,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/f2a07ad17c0d4d568bded44e4ba83d30 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/f2a07ad17c0d4d568bded44e4ba83d30 2024-11-11T06:41:46,914 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24ed63535f254a12993bcfbbdd67f140 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24ed63535f254a12993bcfbbdd67f140 2024-11-11T06:41:46,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/691c8f17b0294d9c94ecfd793b93374c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/691c8f17b0294d9c94ecfd793b93374c 2024-11-11T06:41:46,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/32449970c68742388e864686569d6d67 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/32449970c68742388e864686569d6d67 2024-11-11T06:41:46,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/c6bd93891c7d4503981a9647920faaff to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/c6bd93891c7d4503981a9647920faaff 2024-11-11T06:41:46,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e94d67b461c246cab26a068dc41183ac to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e94d67b461c246cab26a068dc41183ac 2024-11-11T06:41:46,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ae0017633704c33a0c0afbae3cc5f26 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ae0017633704c33a0c0afbae3cc5f26 2024-11-11T06:41:46,920 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/59bdc12f30954bab8f5fbd3f5c602c88, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/170ec40573de49f396bed16936260ae7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19c1cb35e39042eab0e0aca7378791cf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/04a6aa8212f54acc9cc73e1475797f27, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/ce664a231aec4445a7b8d5aae00b4df0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2d15c5239654424182063de27fc6d7a7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/98ec7e64b55841449932b88449ba3fb4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/0b9a2bf649b7478e989eb8a3a75b1263, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/1d26248d4fa44d80b388b1ad9484b14a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/bc875f3598bb4f9f8011600132807710, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/311bdf92e3bb4d10923151c340789076, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/f9543ce2d6174987a3c2cda3ba4ad88e] to archive 2024-11-11T06:41:46,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:41:46,922 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/59bdc12f30954bab8f5fbd3f5c602c88 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/59bdc12f30954bab8f5fbd3f5c602c88 2024-11-11T06:41:46,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/170ec40573de49f396bed16936260ae7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/170ec40573de49f396bed16936260ae7 2024-11-11T06:41:46,924 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19c1cb35e39042eab0e0aca7378791cf to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19c1cb35e39042eab0e0aca7378791cf 2024-11-11T06:41:46,925 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/04a6aa8212f54acc9cc73e1475797f27 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/04a6aa8212f54acc9cc73e1475797f27 2024-11-11T06:41:46,926 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/ce664a231aec4445a7b8d5aae00b4df0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/ce664a231aec4445a7b8d5aae00b4df0 2024-11-11T06:41:46,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2d15c5239654424182063de27fc6d7a7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2d15c5239654424182063de27fc6d7a7 2024-11-11T06:41:46,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/98ec7e64b55841449932b88449ba3fb4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/98ec7e64b55841449932b88449ba3fb4 2024-11-11T06:41:46,928 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/0b9a2bf649b7478e989eb8a3a75b1263 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/0b9a2bf649b7478e989eb8a3a75b1263 2024-11-11T06:41:46,929 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/1d26248d4fa44d80b388b1ad9484b14a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/1d26248d4fa44d80b388b1ad9484b14a 2024-11-11T06:41:46,930 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/bc875f3598bb4f9f8011600132807710 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/bc875f3598bb4f9f8011600132807710 2024-11-11T06:41:46,931 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/311bdf92e3bb4d10923151c340789076 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/311bdf92e3bb4d10923151c340789076 2024-11-11T06:41:46,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/f9543ce2d6174987a3c2cda3ba4ad88e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/f9543ce2d6174987a3c2cda3ba4ad88e 2024-11-11T06:41:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T06:41:47,213 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/50515f2bedee4f7789b370eb59e9e8d2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/50515f2bedee4f7789b370eb59e9e8d2 2024-11-11T06:41:47,219 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/B of 1a604f2419334e3a9e7d9b217f70c4d8 into 50515f2bedee4f7789b370eb59e9e8d2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:47,219 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:47,219 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/B, priority=13, startTime=1731307306792; duration=0sec 2024-11-11T06:41:47,219 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:47,219 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:B 2024-11-11T06:41:47,219 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:47,220 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:47,220 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:41:47,221 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/C is initiating minor compaction (all files) 2024-11-11T06:41:47,221 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/C in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:47,221 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/699e9a2b9bcb426c9174765f4962edcf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19a4c179014c4c538525ef43703b0e79, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2bf4986361a04e06a5294b67fd80cf32] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=36.0 K 2024-11-11T06:41:47,221 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 699e9a2b9bcb426c9174765f4962edcf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307302643 2024-11-11T06:41:47,221 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/862faec26e4e47f3850a9a17c61b59d7 is 175, key is test_row_0/A:col10/1731307305500/Put/seqid=0 2024-11-11T06:41:47,222 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a4c179014c4c538525ef43703b0e79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731307303316 2024-11-11T06:41:47,222 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf4986361a04e06a5294b67fd80cf32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307305500 2024-11-11T06:41:47,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742209_1385 (size=31208) 2024-11-11T06:41:47,229 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:41:47,230 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/a258a129a095461cb1e66c7d7475a69c is 50, key is test_row_0/C:col10/1731307305500/Put/seqid=0 2024-11-11T06:41:47,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742210_1386 (size=12254) 2024-11-11T06:41:47,238 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115b86b17023ee4259afd2d63fd1d592ff_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115b86b17023ee4259afd2d63fd1d592ff_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:47,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a6bb04fee1ea40d9b0ede64b411889a3, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:47,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a6bb04fee1ea40d9b0ede64b411889a3 is 175, key is test_row_0/A:col10/1731307305606/Put/seqid=0 2024-11-11T06:41:47,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742211_1387 (size=31105) 2024-11-11T06:41:47,245 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a6bb04fee1ea40d9b0ede64b411889a3 2024-11-11T06:41:47,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/24837bb4380846f096aac78592d01eb0 is 50, key is test_row_0/B:col10/1731307305606/Put/seqid=0 2024-11-11T06:41:47,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742212_1388 (size=12151) 2024-11-11T06:41:47,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/24837bb4380846f096aac78592d01eb0 2024-11-11T06:41:47,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/819a6f913a8f4f27bd57224ababf1435 is 50, key is test_row_0/C:col10/1731307305606/Put/seqid=0 2024-11-11T06:41:47,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742213_1389 (size=12151) 2024-11-11T06:41:47,629 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/862faec26e4e47f3850a9a17c61b59d7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/862faec26e4e47f3850a9a17c61b59d7 2024-11-11T06:41:47,633 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/A of 1a604f2419334e3a9e7d9b217f70c4d8 into 862faec26e4e47f3850a9a17c61b59d7(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:47,633 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:47,633 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/A, priority=13, startTime=1731307306792; duration=0sec 2024-11-11T06:41:47,633 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:47,633 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:A 2024-11-11T06:41:47,640 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/a258a129a095461cb1e66c7d7475a69c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/a258a129a095461cb1e66c7d7475a69c 2024-11-11T06:41:47,643 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/C of 1a604f2419334e3a9e7d9b217f70c4d8 into a258a129a095461cb1e66c7d7475a69c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:47,643 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:47,643 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/C, priority=13, startTime=1731307306792; duration=0sec 2024-11-11T06:41:47,643 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:47,643 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:C 2024-11-11T06:41:47,681 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/819a6f913a8f4f27bd57224ababf1435 2024-11-11T06:41:47,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a6bb04fee1ea40d9b0ede64b411889a3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a6bb04fee1ea40d9b0ede64b411889a3 2024-11-11T06:41:47,688 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a6bb04fee1ea40d9b0ede64b411889a3, entries=150, sequenceid=233, filesize=30.4 K 2024-11-11T06:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/24837bb4380846f096aac78592d01eb0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24837bb4380846f096aac78592d01eb0 2024-11-11T06:41:47,691 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24837bb4380846f096aac78592d01eb0, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T06:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/819a6f913a8f4f27bd57224ababf1435 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/819a6f913a8f4f27bd57224ababf1435 2024-11-11T06:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,695 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/819a6f913a8f4f27bd57224ababf1435, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T06:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,731 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 1a604f2419334e3a9e7d9b217f70c4d8 in 933ms, sequenceid=233, compaction requested=false 2024-11-11T06:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:47,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:47,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-11T06:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-11T06:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T06:41:47,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6990 sec 2024-11-11T06:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7050 sec 2024-11-11T06:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110f950b13e1bc427997947ab4c6796285_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307307888/Put/seqid=0 2024-11-11T06:41:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742215_1391 (size=24758) 2024-11-11T06:41:47,930 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:47,935 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110f950b13e1bc427997947ab4c6796285_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110f950b13e1bc427997947ab4c6796285_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:47,936 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/d869bfc81ea04fae9bee8bf617c72358, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:47,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/d869bfc81ea04fae9bee8bf617c72358 is 175, key is test_row_0/A:col10/1731307307888/Put/seqid=0 2024-11-11T06:41:47,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742214_1390 (size=74394) 2024-11-11T06:41:47,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307367960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:47,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307367961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:47,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307367964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:47,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307367971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307368079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307368080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307368080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307368089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T06:41:48,189 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:48,189 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T06:41:48,191 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T06:41:48,191 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:48,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T06:41:48,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307368299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307368299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307368299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307368309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,339 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/d869bfc81ea04fae9bee8bf617c72358 2024-11-11T06:41:48,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T06:41:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/348cc3e2a0314562ae2f76fc5f322e27 is 50, key is test_row_0/B:col10/1731307307888/Put/seqid=0 2024-11-11T06:41:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742216_1392 (size=12151) 2024-11-11T06:41:48,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T06:41:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:48,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T06:41:48,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307368608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307368609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307368609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307368619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:48,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:48,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T06:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:48,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/348cc3e2a0314562ae2f76fc5f322e27 2024-11-11T06:41:48,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/7964c3216f3f4c9d9fdc2c363afb1678 is 50, key is test_row_0/C:col10/1731307307888/Put/seqid=0 2024-11-11T06:41:48,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742217_1393 (size=12151) 2024-11-11T06:41:48,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/7964c3216f3f4c9d9fdc2c363afb1678 2024-11-11T06:41:48,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/d869bfc81ea04fae9bee8bf617c72358 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d869bfc81ea04fae9bee8bf617c72358 2024-11-11T06:41:48,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d869bfc81ea04fae9bee8bf617c72358, entries=400, sequenceid=247, filesize=72.7 K 2024-11-11T06:41:48,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/348cc3e2a0314562ae2f76fc5f322e27 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/348cc3e2a0314562ae2f76fc5f322e27 2024-11-11T06:41:48,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/348cc3e2a0314562ae2f76fc5f322e27, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T06:41:48,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/7964c3216f3f4c9d9fdc2c363afb1678 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/7964c3216f3f4c9d9fdc2c363afb1678 2024-11-11T06:41:48,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/7964c3216f3f4c9d9fdc2c363afb1678, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T06:41:48,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1a604f2419334e3a9e7d9b217f70c4d8 in 881ms, sequenceid=247, compaction requested=true 2024-11-11T06:41:48,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:48,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:48,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:48,772 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:48,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:48,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:48,772 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:48,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:48,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:48,772 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136707 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:48,772 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:48,772 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/A is initiating minor compaction (all files) 2024-11-11T06:41:48,772 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/B is initiating minor compaction (all files) 2024-11-11T06:41:48,772 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/A in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,772 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/B in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,773 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/50515f2bedee4f7789b370eb59e9e8d2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24837bb4380846f096aac78592d01eb0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/348cc3e2a0314562ae2f76fc5f322e27] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=36.1 K 2024-11-11T06:41:48,773 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/862faec26e4e47f3850a9a17c61b59d7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a6bb04fee1ea40d9b0ede64b411889a3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d869bfc81ea04fae9bee8bf617c72358] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=133.5 K 2024-11-11T06:41:48,773 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,773 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/862faec26e4e47f3850a9a17c61b59d7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a6bb04fee1ea40d9b0ede64b411889a3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d869bfc81ea04fae9bee8bf617c72358] 2024-11-11T06:41:48,773 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 50515f2bedee4f7789b370eb59e9e8d2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307305500 2024-11-11T06:41:48,773 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 862faec26e4e47f3850a9a17c61b59d7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307305500 2024-11-11T06:41:48,773 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 24837bb4380846f096aac78592d01eb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731307305592 2024-11-11T06:41:48,773 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6bb04fee1ea40d9b0ede64b411889a3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731307305592 2024-11-11T06:41:48,773 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 348cc3e2a0314562ae2f76fc5f322e27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307307874 2024-11-11T06:41:48,773 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting d869bfc81ea04fae9bee8bf617c72358, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307307845 2024-11-11T06:41:48,777 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:48,778 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:41:48,779 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/7061c9457f4c4dec8ada48a979d76d3c is 50, key is test_row_0/B:col10/1731307307888/Put/seqid=0 2024-11-11T06:41:48,779 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115e12097091c847cc984a9e1e2c8e9e84_1a604f2419334e3a9e7d9b217f70c4d8 store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:48,782 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115e12097091c847cc984a9e1e2c8e9e84_1a604f2419334e3a9e7d9b217f70c4d8, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:48,782 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115e12097091c847cc984a9e1e2c8e9e84_1a604f2419334e3a9e7d9b217f70c4d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:48,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742218_1394 (size=12357) 2024-11-11T06:41:48,791 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/7061c9457f4c4dec8ada48a979d76d3c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/7061c9457f4c4dec8ada48a979d76d3c 2024-11-11T06:41:48,795 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/B of 1a604f2419334e3a9e7d9b217f70c4d8 into 7061c9457f4c4dec8ada48a979d76d3c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:48,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742219_1395 (size=4469) 2024-11-11T06:41:48,795 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:48,796 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/B, priority=13, startTime=1731307308772; duration=0sec 2024-11-11T06:41:48,796 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:48,796 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:B 2024-11-11T06:41:48,796 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:48,797 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36556 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:48,797 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/C is initiating minor compaction (all files) 2024-11-11T06:41:48,797 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#A#compaction#328 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:41:48,797 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/C in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,797 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/a258a129a095461cb1e66c7d7475a69c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/819a6f913a8f4f27bd57224ababf1435, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/7964c3216f3f4c9d9fdc2c363afb1678] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=35.7 K 2024-11-11T06:41:48,798 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting a258a129a095461cb1e66c7d7475a69c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307305500 2024-11-11T06:41:48,798 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a5bb44c090754cd3824a5f82e574fc5a is 175, key is test_row_0/A:col10/1731307307888/Put/seqid=0 2024-11-11T06:41:48,798 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 819a6f913a8f4f27bd57224ababf1435, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731307305592 2024-11-11T06:41:48,798 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7964c3216f3f4c9d9fdc2c363afb1678, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307307874 2024-11-11T06:41:48,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742220_1396 (size=31311) 2024-11-11T06:41:48,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T06:41:48,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:48,802 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:41:48,803 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:41:48,804 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/c3fb0302a3f64628b650cb880aa750f1 is 50, key is test_row_0/C:col10/1731307307888/Put/seqid=0 2024-11-11T06:41:48,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742221_1397 (size=12357) 2024-11-11T06:41:48,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111158101cb75fdf4fd28b490af479cc76c5_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307307961/Put/seqid=0 2024-11-11T06:41:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T06:41:48,820 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/c3fb0302a3f64628b650cb880aa750f1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/c3fb0302a3f64628b650cb880aa750f1 2024-11-11T06:41:48,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742222_1398 (size=12454) 2024-11-11T06:41:48,824 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/C of 1a604f2419334e3a9e7d9b217f70c4d8 into c3fb0302a3f64628b650cb880aa750f1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:48,824 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:48,824 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/C, priority=13, startTime=1731307308772; duration=0sec 2024-11-11T06:41:48,824 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:48,824 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:C 2024-11-11T06:41:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:49,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:49,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307369164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307369165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307369166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307369167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,204 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a5bb44c090754cd3824a5f82e574fc5a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a5bb44c090754cd3824a5f82e574fc5a 2024-11-11T06:41:49,208 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/A of 1a604f2419334e3a9e7d9b217f70c4d8 into a5bb44c090754cd3824a5f82e574fc5a(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:49,208 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:49,208 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/A, priority=13, startTime=1731307308772; duration=0sec 2024-11-11T06:41:49,208 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:49,208 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:A 2024-11-11T06:41:49,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:49,225 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111158101cb75fdf4fd28b490af479cc76c5_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158101cb75fdf4fd28b490af479cc76c5_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:49,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/3de200d380ea46ea9dd5043889b2a7de, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:49,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/3de200d380ea46ea9dd5043889b2a7de is 175, key is test_row_0/A:col10/1731307307961/Put/seqid=0 2024-11-11T06:41:49,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742223_1399 (size=31255) 2024-11-11T06:41:49,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307369269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307369279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307369289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307369289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T06:41:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307369479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307369488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307369499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307369509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,630 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/3de200d380ea46ea9dd5043889b2a7de 2024-11-11T06:41:49,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/a675bc988188467188cc0b5ceb22210b is 50, key is test_row_0/B:col10/1731307307961/Put/seqid=0 2024-11-11T06:41:49,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742224_1400 (size=12301) 2024-11-11T06:41:49,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307369789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307369799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307369809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:49,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307369819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:50,039 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/a675bc988188467188cc0b5ceb22210b 2024-11-11T06:41:50,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/243418f60b1942ef83e817e79e4e82fa is 50, key is test_row_0/C:col10/1731307307961/Put/seqid=0 2024-11-11T06:41:50,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742225_1401 (size=12301) 2024-11-11T06:41:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307370309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:50,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:50,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307370319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:50,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:50,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307370319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:50,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:50,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307370329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:50,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T06:41:50,448 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/243418f60b1942ef83e817e79e4e82fa 2024-11-11T06:41:50,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/3de200d380ea46ea9dd5043889b2a7de as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3de200d380ea46ea9dd5043889b2a7de 2024-11-11T06:41:50,462 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3de200d380ea46ea9dd5043889b2a7de, entries=150, sequenceid=273, filesize=30.5 K 2024-11-11T06:41:50,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/a675bc988188467188cc0b5ceb22210b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/a675bc988188467188cc0b5ceb22210b 2024-11-11T06:41:50,465 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/a675bc988188467188cc0b5ceb22210b, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T06:41:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/243418f60b1942ef83e817e79e4e82fa as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/243418f60b1942ef83e817e79e4e82fa 2024-11-11T06:41:50,469 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/243418f60b1942ef83e817e79e4e82fa, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T06:41:50,470 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1668ms, sequenceid=273, compaction requested=false 2024-11-11T06:41:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T06:41:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T06:41:50,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T06:41:50,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2800 sec 2024-11-11T06:41:50,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2840 sec 2024-11-11T06:41:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:51,333 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:41:51,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:41:51,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118ecaf4a0656a4a86a6105cbcbb00454f_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307311332/Put/seqid=0 2024-11-11T06:41:51,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742226_1402 (size=14994) 2024-11-11T06:41:51,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307371380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307371383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307371389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307371390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307371499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307371499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307371500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307371508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56428 deadline: 1731307371589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,595 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_2' in 1a604f2419334e3a9e7d9b217f70c4d8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:41:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307371709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307371709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307371709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307371719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:51,747 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:51,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118ecaf4a0656a4a86a6105cbcbb00454f_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118ecaf4a0656a4a86a6105cbcbb00454f_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:51,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c0ea1fa81a0a429799015f020ab2c6e6, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:51,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c0ea1fa81a0a429799015f020ab2c6e6 is 175, key is test_row_0/A:col10/1731307311332/Put/seqid=0 2024-11-11T06:41:51,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742227_1403 (size=39949) 2024-11-11T06:41:52,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307372019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:52,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:52,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307372019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:52,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:52,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307372019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:52,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307372039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:52,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c0ea1fa81a0a429799015f020ab2c6e6 2024-11-11T06:41:52,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/cda4c3ed05ea48abace4885eb404ea87 is 50, key is test_row_0/B:col10/1731307311332/Put/seqid=0 2024-11-11T06:41:52,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742228_1404 (size=12301) 2024-11-11T06:41:52,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/cda4c3ed05ea48abace4885eb404ea87 2024-11-11T06:41:52,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/de894674d4d841389776d68b77703eea is 50, key is test_row_0/C:col10/1731307311332/Put/seqid=0 2024-11-11T06:41:52,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742229_1405 (size=12301) 2024-11-11T06:41:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T06:41:52,349 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:41:52,349 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:41:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-11T06:41:52,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:41:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T06:41:52,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:41:52,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:41:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T06:41:52,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T06:41:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:52,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:52,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:52,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:41:52,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56450 deadline: 1731307372529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:52,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56422 deadline: 1731307372529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:52,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56398 deadline: 1731307372538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:52,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:41:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56452 deadline: 1731307372559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:41:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:41:52,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:41:52,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/de894674d4d841389776d68b77703eea 2024-11-11T06:41:52,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/c0ea1fa81a0a429799015f020ab2c6e6 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c0ea1fa81a0a429799015f020ab2c6e6 2024-11-11T06:41:52,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c0ea1fa81a0a429799015f020ab2c6e6, entries=200, sequenceid=287, filesize=39.0 K 2024-11-11T06:41:52,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/cda4c3ed05ea48abace4885eb404ea87 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/cda4c3ed05ea48abace4885eb404ea87 2024-11-11T06:41:52,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/cda4c3ed05ea48abace4885eb404ea87, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T06:41:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/de894674d4d841389776d68b77703eea as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/de894674d4d841389776d68b77703eea 2024-11-11T06:41:52,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/de894674d4d841389776d68b77703eea, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T06:41:52,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1270ms, sequenceid=287, compaction requested=true 2024-11-11T06:41:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:41:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:52,603 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:41:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:52,603 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a604f2419334e3a9e7d9b217f70c4d8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:41:52,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:52,604 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:52,604 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:52,604 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/A is initiating minor compaction (all files) 2024-11-11T06:41:52,604 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/B is initiating minor compaction (all files) 2024-11-11T06:41:52,604 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/A in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:52,604 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/B in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:52,604 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a5bb44c090754cd3824a5f82e574fc5a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3de200d380ea46ea9dd5043889b2a7de, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c0ea1fa81a0a429799015f020ab2c6e6] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=100.1 K 2024-11-11T06:41:52,604 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/7061c9457f4c4dec8ada48a979d76d3c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/a675bc988188467188cc0b5ceb22210b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/cda4c3ed05ea48abace4885eb404ea87] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=36.1 K 2024-11-11T06:41:52,604 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:52,604 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a5bb44c090754cd3824a5f82e574fc5a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3de200d380ea46ea9dd5043889b2a7de, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c0ea1fa81a0a429799015f020ab2c6e6] 2024-11-11T06:41:52,604 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7061c9457f4c4dec8ada48a979d76d3c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307307874 2024-11-11T06:41:52,604 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting a5bb44c090754cd3824a5f82e574fc5a, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307307874 2024-11-11T06:41:52,605 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a675bc988188467188cc0b5ceb22210b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731307307958 2024-11-11T06:41:52,605 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de200d380ea46ea9dd5043889b2a7de, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731307307958 2024-11-11T06:41:52,605 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting cda4c3ed05ea48abace4885eb404ea87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731307309165 2024-11-11T06:41:52,605 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c0ea1fa81a0a429799015f020ab2c6e6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731307309163 2024-11-11T06:41:52,612 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:41:52,612 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/2baa6a4a14fa4bd9bcc6aa18ac017454 is 50, key is test_row_0/B:col10/1731307311332/Put/seqid=0 2024-11-11T06:41:52,613 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:52,619 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111409edb87e0f14ed6b582a2cbf737bbc3_1a604f2419334e3a9e7d9b217f70c4d8 store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:52,621 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111409edb87e0f14ed6b582a2cbf737bbc3_1a604f2419334e3a9e7d9b217f70c4d8, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:52,621 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111409edb87e0f14ed6b582a2cbf737bbc3_1a604f2419334e3a9e7d9b217f70c4d8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:52,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742230_1406 (size=12609) 2024-11-11T06:41:52,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742231_1407 (size=4469) 2024-11-11T06:41:52,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T06:41:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:52,656 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:41:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c1999c0bfcd64da899eaddd2433de0cf_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307311381/Put/seqid=0 2024-11-11T06:41:52,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742232_1408 (size=12454) 2024-11-11T06:41:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T06:41:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T06:41:53,027 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/2baa6a4a14fa4bd9bcc6aa18ac017454 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2baa6a4a14fa4bd9bcc6aa18ac017454 2024-11-11T06:41:53,030 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/B of 1a604f2419334e3a9e7d9b217f70c4d8 into 2baa6a4a14fa4bd9bcc6aa18ac017454(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:53,030 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:53,030 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/B, priority=13, startTime=1731307312603; duration=0sec 2024-11-11T06:41:53,031 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:41:53,031 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:B 2024-11-11T06:41:53,031 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:41:53,032 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:41:53,032 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 1a604f2419334e3a9e7d9b217f70c4d8/C is initiating minor compaction (all files) 2024-11-11T06:41:53,032 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a604f2419334e3a9e7d9b217f70c4d8/C in TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:53,032 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/c3fb0302a3f64628b650cb880aa750f1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/243418f60b1942ef83e817e79e4e82fa, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/de894674d4d841389776d68b77703eea] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp, totalSize=36.1 K 2024-11-11T06:41:53,032 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3fb0302a3f64628b650cb880aa750f1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307307874 2024-11-11T06:41:53,033 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 243418f60b1942ef83e817e79e4e82fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731307307958 2024-11-11T06:41:53,033 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:41:53,033 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting de894674d4d841389776d68b77703eea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731307309165 2024-11-11T06:41:53,033 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/6d665ad92e7442128802dddb2203bfa1 is 175, key is test_row_0/A:col10/1731307311332/Put/seqid=0 2024-11-11T06:41:53,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742233_1409 (size=31563) 2024-11-11T06:41:53,038 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a604f2419334e3a9e7d9b217f70c4d8#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:41:53,039 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/29ef8af0d02a4c6dae298e7e188d7ed4 is 50, key is test_row_0/C:col10/1731307311332/Put/seqid=0 2024-11-11T06:41:53,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742234_1410 (size=12609) 2024-11-11T06:41:53,042 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/6d665ad92e7442128802dddb2203bfa1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/6d665ad92e7442128802dddb2203bfa1 2024-11-11T06:41:53,046 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/A of 1a604f2419334e3a9e7d9b217f70c4d8 into 6d665ad92e7442128802dddb2203bfa1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:53,046 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:53,046 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/A, priority=13, startTime=1731307312603; duration=0sec 2024-11-11T06:41:53,046 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:53,046 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:A 2024-11-11T06:41:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:41:53,069 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c1999c0bfcd64da899eaddd2433de0cf_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c1999c0bfcd64da899eaddd2433de0cf_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:53,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/9855ddf9cfd94141b1d8d879e49d5653, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:41:53,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/9855ddf9cfd94141b1d8d879e49d5653 is 175, key is test_row_0/A:col10/1731307311381/Put/seqid=0 2024-11-11T06:41:53,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742235_1411 (size=31255) 2024-11-11T06:41:53,253 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-11-11T06:41:53,253 INFO [Thread-1495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1495. 2024-11-11T06:41:53,253 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,253 DEBUG [Thread-1495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,253 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,253 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,253 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,253 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,256 INFO [Thread-1493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1493. 2024-11-11T06:41:53,256 DEBUG [Thread-1493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,256 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,256 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,258 INFO [Thread-1497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1497. 2024-11-11T06:41:53,258 DEBUG [Thread-1497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,258 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,258 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,259 INFO [Thread-1491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1491. 2024-11-11T06:41:53,259 DEBUG [Thread-1491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,259 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,259 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,453 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/29ef8af0d02a4c6dae298e7e188d7ed4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/29ef8af0d02a4c6dae298e7e188d7ed4 2024-11-11T06:41:53,461 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a604f2419334e3a9e7d9b217f70c4d8/C of 1a604f2419334e3a9e7d9b217f70c4d8 into 29ef8af0d02a4c6dae298e7e188d7ed4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:41:53,462 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:53,462 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., storeName=1a604f2419334e3a9e7d9b217f70c4d8/C, priority=13, startTime=1731307312603; duration=0sec 2024-11-11T06:41:53,462 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:41:53,462 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a604f2419334e3a9e7d9b217f70c4d8:C 2024-11-11T06:41:53,474 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/9855ddf9cfd94141b1d8d879e49d5653 2024-11-11T06:41:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/d76c224b83bb46aaa16d0839a7353663 is 50, key is test_row_0/B:col10/1731307311381/Put/seqid=0 2024-11-11T06:41:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T06:41:53,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742236_1412 (size=12301) 2024-11-11T06:41:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:41:53,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. as already flushing 2024-11-11T06:41:53,589 INFO [Thread-1480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1480. 2024-11-11T06:41:53,589 INFO [Thread-1486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1486. 2024-11-11T06:41:53,589 INFO [Thread-1484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1484. 2024-11-11T06:41:53,589 DEBUG [Thread-1486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,589 DEBUG [Thread-1480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,589 DEBUG [Thread-1484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,589 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,589 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,589 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,589 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,589 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,589 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,590 INFO [Thread-1482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1482. 2024-11-11T06:41:53,591 DEBUG [Thread-1482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:41:53,591 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,591 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:41:53,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:41:53,892 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/d76c224b83bb46aaa16d0839a7353663 2024-11-11T06:41:53,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/74b2a25b9caf4f14b0ff725dd2d764a4 is 50, key is test_row_0/C:col10/1731307311381/Put/seqid=0 2024-11-11T06:41:53,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742237_1413 (size=12301) 2024-11-11T06:41:54,315 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/74b2a25b9caf4f14b0ff725dd2d764a4 2024-11-11T06:41:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/9855ddf9cfd94141b1d8d879e49d5653 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9855ddf9cfd94141b1d8d879e49d5653 2024-11-11T06:41:54,329 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9855ddf9cfd94141b1d8d879e49d5653, entries=150, sequenceid=312, filesize=30.5 K 2024-11-11T06:41:54,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/d76c224b83bb46aaa16d0839a7353663 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/d76c224b83bb46aaa16d0839a7353663 2024-11-11T06:41:54,335 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/d76c224b83bb46aaa16d0839a7353663, entries=150, sequenceid=312, filesize=12.0 K 2024-11-11T06:41:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/74b2a25b9caf4f14b0ff725dd2d764a4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/74b2a25b9caf4f14b0ff725dd2d764a4 2024-11-11T06:41:54,341 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/74b2a25b9caf4f14b0ff725dd2d764a4, entries=150, sequenceid=312, filesize=12.0 K 2024-11-11T06:41:54,342 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1685ms, sequenceid=312, compaction requested=false 2024-11-11T06:41:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 1a604f2419334e3a9e7d9b217f70c4d8: 2024-11-11T06:41:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:41:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-11T06:41:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-11T06:41:54,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T06:41:54,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9920 sec 2024-11-11T06:41:54,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.9960 sec 2024-11-11T06:41:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T06:41:54,511 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:01,674 INFO [Thread-1488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1488. 2024-11-11T06:42:01,674 DEBUG [Thread-1488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:01,674 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:01,675 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:01,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:01,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T06:42:01,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-11T06:42:01,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1448 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4342 rows 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1449 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4343 rows 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1455 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4365 rows 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1438 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4313 rows 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1452 2024-11-11T06:42:01,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4354 rows 2024-11-11T06:42:01,677 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T06:42:01,677 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:42:01,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:01,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:01,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:01,680 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T06:42:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T06:42:01,683 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307321683"}]},"ts":"1731307321683"} 2024-11-11T06:42:01,685 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T06:42:01,685 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T06:42:01,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:42:01,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, UNASSIGN}] 2024-11-11T06:42:01,689 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, UNASSIGN 2024-11-11T06:42:01,689 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=1a604f2419334e3a9e7d9b217f70c4d8, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:01,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, UNASSIGN because future has completed 2024-11-11T06:42:01,691 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:42:01,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:42:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T06:42:01,844 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:01,844 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:42:01,845 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 1a604f2419334e3a9e7d9b217f70c4d8, disabling compactions & flushes 2024-11-11T06:42:01,845 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:42:01,845 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:42:01,845 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. after waiting 0 ms 2024-11-11T06:42:01,845 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:42:01,845 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 1a604f2419334e3a9e7d9b217f70c4d8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T06:42:01,857 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a10adbcb2583474b8f7953929c11e825_1a604f2419334e3a9e7d9b217f70c4d8 is 50, key is test_row_0/A:col10/1731307313550/Put/seqid=0 2024-11-11T06:42:01,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742238_1414 (size=12454) 2024-11-11T06:42:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T06:42:02,262 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:02,271 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a10adbcb2583474b8f7953929c11e825_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a10adbcb2583474b8f7953929c11e825_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:02,272 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a423dbb1001f4095950eab8706f57142, store: [table=TestAcidGuarantees family=A region=1a604f2419334e3a9e7d9b217f70c4d8] 2024-11-11T06:42:02,273 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a423dbb1001f4095950eab8706f57142 is 175, key is test_row_0/A:col10/1731307313550/Put/seqid=0 2024-11-11T06:42:02,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742239_1415 (size=31255) 2024-11-11T06:42:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T06:42:02,677 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a423dbb1001f4095950eab8706f57142 2024-11-11T06:42:02,689 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/5ffdcc6a31a04ad095f4a20fb7ed45c0 is 50, key is test_row_0/B:col10/1731307313550/Put/seqid=0 2024-11-11T06:42:02,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742240_1416 (size=12301) 2024-11-11T06:42:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T06:42:03,094 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/5ffdcc6a31a04ad095f4a20fb7ed45c0 2024-11-11T06:42:03,106 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/8b9607e9fb314788b7caf40e5f57dd49 is 50, key is test_row_0/C:col10/1731307313550/Put/seqid=0 2024-11-11T06:42:03,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742241_1417 (size=12301) 2024-11-11T06:42:03,513 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/8b9607e9fb314788b7caf40e5f57dd49 2024-11-11T06:42:03,524 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/A/a423dbb1001f4095950eab8706f57142 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a423dbb1001f4095950eab8706f57142 2024-11-11T06:42:03,528 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a423dbb1001f4095950eab8706f57142, entries=150, sequenceid=323, filesize=30.5 K 2024-11-11T06:42:03,529 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/B/5ffdcc6a31a04ad095f4a20fb7ed45c0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ffdcc6a31a04ad095f4a20fb7ed45c0 2024-11-11T06:42:03,533 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ffdcc6a31a04ad095f4a20fb7ed45c0, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T06:42:03,533 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/.tmp/C/8b9607e9fb314788b7caf40e5f57dd49 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/8b9607e9fb314788b7caf40e5f57dd49 2024-11-11T06:42:03,537 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/8b9607e9fb314788b7caf40e5f57dd49, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T06:42:03,538 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1693ms, sequenceid=323, compaction requested=true 2024-11-11T06:42:03,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9e65ee9838414cb28e45ca008a9db4c2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/ed00493fb600494484e4d24053c57f0b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/5fa5ed39ce2a4df1a7642d077e961e1a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/862faec26e4e47f3850a9a17c61b59d7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a6bb04fee1ea40d9b0ede64b411889a3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d869bfc81ea04fae9bee8bf617c72358, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a5bb44c090754cd3824a5f82e574fc5a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3de200d380ea46ea9dd5043889b2a7de, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c0ea1fa81a0a429799015f020ab2c6e6] to archive 2024-11-11T06:42:03,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:03,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9e65ee9838414cb28e45ca008a9db4c2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9e65ee9838414cb28e45ca008a9db4c2 2024-11-11T06:42:03,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/ed00493fb600494484e4d24053c57f0b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/ed00493fb600494484e4d24053c57f0b 2024-11-11T06:42:03,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/5fa5ed39ce2a4df1a7642d077e961e1a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/5fa5ed39ce2a4df1a7642d077e961e1a 2024-11-11T06:42:03,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/862faec26e4e47f3850a9a17c61b59d7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/862faec26e4e47f3850a9a17c61b59d7 2024-11-11T06:42:03,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a6bb04fee1ea40d9b0ede64b411889a3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a6bb04fee1ea40d9b0ede64b411889a3 2024-11-11T06:42:03,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d869bfc81ea04fae9bee8bf617c72358 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/d869bfc81ea04fae9bee8bf617c72358 2024-11-11T06:42:03,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a5bb44c090754cd3824a5f82e574fc5a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a5bb44c090754cd3824a5f82e574fc5a 2024-11-11T06:42:03,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3de200d380ea46ea9dd5043889b2a7de to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/3de200d380ea46ea9dd5043889b2a7de 2024-11-11T06:42:03,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c0ea1fa81a0a429799015f020ab2c6e6 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/c0ea1fa81a0a429799015f020ab2c6e6 2024-11-11T06:42:03,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e68a71ffb50f436486442c908fc65fb4, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/058f887b2694428c8ca6110f126ef87b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/50515f2bedee4f7789b370eb59e9e8d2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2981c6951df24b5cb0cff6fd2c9dab40, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24837bb4380846f096aac78592d01eb0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/7061c9457f4c4dec8ada48a979d76d3c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/348cc3e2a0314562ae2f76fc5f322e27, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/a675bc988188467188cc0b5ceb22210b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/cda4c3ed05ea48abace4885eb404ea87] to archive 2024-11-11T06:42:03,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:03,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e68a71ffb50f436486442c908fc65fb4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/e68a71ffb50f436486442c908fc65fb4 2024-11-11T06:42:03,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/058f887b2694428c8ca6110f126ef87b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/058f887b2694428c8ca6110f126ef87b 2024-11-11T06:42:03,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/50515f2bedee4f7789b370eb59e9e8d2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/50515f2bedee4f7789b370eb59e9e8d2 2024-11-11T06:42:03,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2981c6951df24b5cb0cff6fd2c9dab40 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2981c6951df24b5cb0cff6fd2c9dab40 2024-11-11T06:42:03,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24837bb4380846f096aac78592d01eb0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/24837bb4380846f096aac78592d01eb0 2024-11-11T06:42:03,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/7061c9457f4c4dec8ada48a979d76d3c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/7061c9457f4c4dec8ada48a979d76d3c 2024-11-11T06:42:03,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/348cc3e2a0314562ae2f76fc5f322e27 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/348cc3e2a0314562ae2f76fc5f322e27 2024-11-11T06:42:03,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/a675bc988188467188cc0b5ceb22210b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/a675bc988188467188cc0b5ceb22210b 2024-11-11T06:42:03,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/cda4c3ed05ea48abace4885eb404ea87 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/cda4c3ed05ea48abace4885eb404ea87 2024-11-11T06:42:03,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/699e9a2b9bcb426c9174765f4962edcf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19a4c179014c4c538525ef43703b0e79, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/a258a129a095461cb1e66c7d7475a69c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2bf4986361a04e06a5294b67fd80cf32, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/819a6f913a8f4f27bd57224ababf1435, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/c3fb0302a3f64628b650cb880aa750f1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/7964c3216f3f4c9d9fdc2c363afb1678, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/243418f60b1942ef83e817e79e4e82fa, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/de894674d4d841389776d68b77703eea] to archive 2024-11-11T06:42:03,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:03,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/699e9a2b9bcb426c9174765f4962edcf to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/699e9a2b9bcb426c9174765f4962edcf 2024-11-11T06:42:03,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19a4c179014c4c538525ef43703b0e79 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/19a4c179014c4c538525ef43703b0e79 2024-11-11T06:42:03,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/a258a129a095461cb1e66c7d7475a69c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/a258a129a095461cb1e66c7d7475a69c 2024-11-11T06:42:03,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2bf4986361a04e06a5294b67fd80cf32 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/2bf4986361a04e06a5294b67fd80cf32 2024-11-11T06:42:03,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/819a6f913a8f4f27bd57224ababf1435 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/819a6f913a8f4f27bd57224ababf1435 2024-11-11T06:42:03,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/c3fb0302a3f64628b650cb880aa750f1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/c3fb0302a3f64628b650cb880aa750f1 2024-11-11T06:42:03,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/7964c3216f3f4c9d9fdc2c363afb1678 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/7964c3216f3f4c9d9fdc2c363afb1678 2024-11-11T06:42:03,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/243418f60b1942ef83e817e79e4e82fa to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/243418f60b1942ef83e817e79e4e82fa 2024-11-11T06:42:03,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/de894674d4d841389776d68b77703eea to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/de894674d4d841389776d68b77703eea 2024-11-11T06:42:03,582 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-11-11T06:42:03,582 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. 2024-11-11T06:42:03,582 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 1a604f2419334e3a9e7d9b217f70c4d8: Waiting for close lock at 1731307321845Running coprocessor pre-close hooks at 1731307321845Disabling compacts and flushes for region at 1731307321845Disabling writes for close at 1731307321845Obtaining lock to block concurrent updates at 1731307321845Preparing flush snapshotting stores in 1a604f2419334e3a9e7d9b217f70c4d8 at 1731307321845Finished memstore snapshotting TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731307321846 (+1 ms)Flushing stores of TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8. at 1731307321847 (+1 ms)Flushing 1a604f2419334e3a9e7d9b217f70c4d8/A: creating writer at 1731307321848 (+1 ms)Flushing mob file 1a604f2419334e3a9e7d9b217f70c4d8/A: appending metadata at 1731307321857 (+9 ms)Flushing mob file 1a604f2419334e3a9e7d9b217f70c4d8/A: closing flushed file at 1731307321857Flushing 1a604f2419334e3a9e7d9b217f70c4d8/A: appending metadata at 1731307322272 (+415 ms)Flushing 1a604f2419334e3a9e7d9b217f70c4d8/A: closing flushed file at 1731307322272Flushing 1a604f2419334e3a9e7d9b217f70c4d8/B: creating writer at 1731307322684 (+412 ms)Flushing 1a604f2419334e3a9e7d9b217f70c4d8/B: appending metadata at 1731307322688 (+4 ms)Flushing 1a604f2419334e3a9e7d9b217f70c4d8/B: closing flushed file at 1731307322689 (+1 ms)Flushing 1a604f2419334e3a9e7d9b217f70c4d8/C: creating writer at 1731307323102 (+413 ms)Flushing 1a604f2419334e3a9e7d9b217f70c4d8/C: appending metadata at 1731307323106 (+4 ms)Flushing 1a604f2419334e3a9e7d9b217f70c4d8/C: closing flushed file at 1731307323106Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37f4e23b: reopening flushed file at 1731307323523 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d1883e: reopening flushed file at 1731307323528 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d8eefdb: reopening flushed file at 1731307323533 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1a604f2419334e3a9e7d9b217f70c4d8 in 1693ms, sequenceid=323, compaction requested=true at 1731307323538 (+5 ms)Writing region close event to WAL at 1731307323580 (+42 ms)Running coprocessor post-close hooks at 1731307323582 (+2 ms)Closed at 1731307323582 2024-11-11T06:42:03,584 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,584 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=1a604f2419334e3a9e7d9b217f70c4d8, regionState=CLOSED 2024-11-11T06:42:03,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:42:03,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T06:42:03,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a604f2419334e3a9e7d9b217f70c4d8, server=ce2cf0244921,35807,1731307183669 in 1.8960 sec 2024-11-11T06:42:03,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-11T06:42:03,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a604f2419334e3a9e7d9b217f70c4d8, UNASSIGN in 1.9010 sec 2024-11-11T06:42:03,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T06:42:03,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9050 sec 2024-11-11T06:42:03,595 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307323595"}]},"ts":"1731307323595"} 2024-11-11T06:42:03,597 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T06:42:03,597 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T06:42:03,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-11T06:42:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T06:42:03,829 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:03,829 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T06:42:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:03,831 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T06:42:03,831 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:03,834 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,836 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/recovered.edits] 2024-11-11T06:42:03,838 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/6d665ad92e7442128802dddb2203bfa1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/6d665ad92e7442128802dddb2203bfa1 2024-11-11T06:42:03,839 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9855ddf9cfd94141b1d8d879e49d5653 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/9855ddf9cfd94141b1d8d879e49d5653 2024-11-11T06:42:03,841 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a423dbb1001f4095950eab8706f57142 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/A/a423dbb1001f4095950eab8706f57142 2024-11-11T06:42:03,843 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2baa6a4a14fa4bd9bcc6aa18ac017454 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/2baa6a4a14fa4bd9bcc6aa18ac017454 2024-11-11T06:42:03,845 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ffdcc6a31a04ad095f4a20fb7ed45c0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/5ffdcc6a31a04ad095f4a20fb7ed45c0 2024-11-11T06:42:03,846 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/d76c224b83bb46aaa16d0839a7353663 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/B/d76c224b83bb46aaa16d0839a7353663 2024-11-11T06:42:03,848 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/29ef8af0d02a4c6dae298e7e188d7ed4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/29ef8af0d02a4c6dae298e7e188d7ed4 2024-11-11T06:42:03,849 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/74b2a25b9caf4f14b0ff725dd2d764a4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/74b2a25b9caf4f14b0ff725dd2d764a4 2024-11-11T06:42:03,850 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/8b9607e9fb314788b7caf40e5f57dd49 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/C/8b9607e9fb314788b7caf40e5f57dd49 2024-11-11T06:42:03,852 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/recovered.edits/326.seqid to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8/recovered.edits/326.seqid 2024-11-11T06:42:03,853 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,853 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T06:42:03,853 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T06:42:03,854 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T06:42:03,856 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110f950b13e1bc427997947ab4c6796285_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110f950b13e1bc427997947ab4c6796285_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,857 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111111eb29ee2d0d4d63bc0fca60534ee8c7_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111111eb29ee2d0d4d63bc0fca60534ee8c7_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,858 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113ecb96a05f66434aa9f2c35225c4b109_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113ecb96a05f66434aa9f2c35225c4b109_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,859 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114bd78bb7a3a24a469fd88b0bd2e2f353_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114bd78bb7a3a24a469fd88b0bd2e2f353_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,860 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158101cb75fdf4fd28b490af479cc76c5_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158101cb75fdf4fd28b490af479cc76c5_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,861 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a0c194dd1724e7eb382cc6f2a052614_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a0c194dd1724e7eb382cc6f2a052614_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,862 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115b86b17023ee4259afd2d63fd1d592ff_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115b86b17023ee4259afd2d63fd1d592ff_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,863 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116532dbfe105d481ab1cc9cd118be3a76_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116532dbfe105d481ab1cc9cd118be3a76_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,863 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117218b25ad9d347f7ad318ece1659121d_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117218b25ad9d347f7ad318ece1659121d_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,864 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118ecaf4a0656a4a86a6105cbcbb00454f_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118ecaf4a0656a4a86a6105cbcbb00454f_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,865 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a10adbcb2583474b8f7953929c11e825_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a10adbcb2583474b8f7953929c11e825_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,866 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c1999c0bfcd64da899eaddd2433de0cf_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c1999c0bfcd64da899eaddd2433de0cf_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,866 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc23ec8cc70e4a5bb92cb739b4bbaf68_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc23ec8cc70e4a5bb92cb739b4bbaf68_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,867 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d0c5a4aaf45b419eb25a6d57f16031a0_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d0c5a4aaf45b419eb25a6d57f16031a0_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,868 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d52213f141504dff91b9524a57f02111_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d52213f141504dff91b9524a57f02111_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,869 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e8ad27022e1b4488b60a10b4a38262e3_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e8ad27022e1b4488b60a10b4a38262e3_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,870 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fd999bdc56a74d5296473638e048c4ac_1a604f2419334e3a9e7d9b217f70c4d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fd999bdc56a74d5296473638e048c4ac_1a604f2419334e3a9e7d9b217f70c4d8 2024-11-11T06:42:03,870 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T06:42:03,872 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:03,873 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T06:42:03,875 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T06:42:03,876 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:03,876 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T06:42:03,876 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731307323876"}]},"ts":"9223372036854775807"} 2024-11-11T06:42:03,877 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T06:42:03,877 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1a604f2419334e3a9e7d9b217f70c4d8, NAME => 'TestAcidGuarantees,,1731307290036.1a604f2419334e3a9e7d9b217f70c4d8.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T06:42:03,877 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T06:42:03,878 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731307323878"}]},"ts":"9223372036854775807"} 2024-11-11T06:42:03,879 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T06:42:03,879 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:03,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-11T06:42:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T06:42:03,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T06:42:03,939 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:03,948 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 237) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=382 (was 423), ProcessCount=11 (was 11), AvailableMemoryMB=10866 (was 10929) 2024-11-11T06:42:03,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=382, ProcessCount=11, AvailableMemoryMB=10866 2024-11-11T06:42:03,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:42:03,957 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:42:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:03,959 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T06:42:03,959 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:03,959 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-11T06:42:03,960 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T06:42:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T06:42:03,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742242_1418 (size=959) 2024-11-11T06:42:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T06:42:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T06:42:04,370 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 2024-11-11T06:42:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742243_1419 (size=53) 2024-11-11T06:42:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T06:42:04,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:42:04,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c363d62112ab347adf2457471c10bf81, disabling compactions & flushes 2024-11-11T06:42:04,817 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:04,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:04,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. after waiting 0 ms 2024-11-11T06:42:04,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:04,817 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:04,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c363d62112ab347adf2457471c10bf81: Waiting for close lock at 1731307324817Disabling compacts and flushes for region at 1731307324817Disabling writes for close at 1731307324817Writing region close event to WAL at 1731307324817Closed at 1731307324817 2024-11-11T06:42:04,820 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T06:42:04,821 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731307324821"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731307324821"}]},"ts":"1731307324821"} 2024-11-11T06:42:04,825 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T06:42:04,827 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T06:42:04,827 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307324827"}]},"ts":"1731307324827"} 2024-11-11T06:42:04,828 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T06:42:04,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, ASSIGN}] 2024-11-11T06:42:04,830 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, ASSIGN 2024-11-11T06:42:04,831 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, ASSIGN; state=OFFLINE, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=false 2024-11-11T06:42:04,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=c363d62112ab347adf2457471c10bf81, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:04,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, ASSIGN because future has completed 2024-11-11T06:42:04,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:42:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T06:42:05,149 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:05,149 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:42:05,150 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,151 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:42:05,151 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,151 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,154 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,156 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c363d62112ab347adf2457471c10bf81 columnFamilyName A 2024-11-11T06:42:05,156 DEBUG [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:05,157 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] regionserver.HStore(327): Store=c363d62112ab347adf2457471c10bf81/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:05,157 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,159 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c363d62112ab347adf2457471c10bf81 columnFamilyName B 2024-11-11T06:42:05,159 DEBUG [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:05,159 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] regionserver.HStore(327): Store=c363d62112ab347adf2457471c10bf81/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:05,159 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,160 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c363d62112ab347adf2457471c10bf81 columnFamilyName C 2024-11-11T06:42:05,160 DEBUG [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:05,160 INFO [StoreOpener-c363d62112ab347adf2457471c10bf81-1 {}] regionserver.HStore(327): Store=c363d62112ab347adf2457471c10bf81/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:05,160 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,161 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,161 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,162 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,162 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,162 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:42:05,163 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,164 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T06:42:05,165 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened c363d62112ab347adf2457471c10bf81; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69503738, jitterRate=0.03568640351295471}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:42:05,165 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:05,165 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for c363d62112ab347adf2457471c10bf81: Running coprocessor pre-open hook at 1731307325151Writing region info on filesystem at 1731307325151Initializing all the Stores at 1731307325153 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307325153Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307325153Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307325153Cleaning up temporary data from old regions at 1731307325162 (+9 ms)Running coprocessor post-open hooks at 1731307325165 (+3 ms)Region opened successfully at 1731307325165 2024-11-11T06:42:05,166 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., pid=118, masterSystemTime=1731307325142 2024-11-11T06:42:05,167 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:05,167 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:05,167 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=c363d62112ab347adf2457471c10bf81, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:05,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:42:05,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T06:42:05,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 in 182 msec 2024-11-11T06:42:05,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-11T06:42:05,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, ASSIGN in 342 msec 2024-11-11T06:42:05,173 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T06:42:05,173 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307325173"}]},"ts":"1731307325173"} 2024-11-11T06:42:05,174 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T06:42:05,174 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T06:42:05,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2170 sec 2024-11-11T06:42:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-11T06:42:06,111 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:06,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,116 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c905f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,119 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2953059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,120 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T06:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,40845,1731307182847 2024-11-11T06:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d0b2c31 2024-11-11T06:42:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T06:42:06,125 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T06:42:06,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,128 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3771d751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402c710a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67494fc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,133 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8791aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,136 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e4a3100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,138 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c95c76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,140 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a44371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,142 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42fbaca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,144 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522b380f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,145 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2b2f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,147 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239957eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,148 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@583cb70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,150 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ebd670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,151 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b05e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,152 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5d768b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bfb49e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,155 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20db2df2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,156 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33071ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:06,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:06,158 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39da6290, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:06,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:06,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:06,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b869189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:06,160 DEBUG [Thread-1913 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,160 DEBUG [Thread-1904 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,160 DEBUG [Thread-1917 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,160 DEBUG [Thread-1902 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:06,161 DEBUG [Thread-1921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,161 DEBUG [Thread-1908 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,162 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,162 DEBUG [Thread-1910 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T06:42:06,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,162 DEBUG [Thread-1906 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T06:42:06,163 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:06,163 DEBUG [Thread-1915 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:06,164 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:06,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,170 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2] 2024-11-11T06:42:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:06,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:06,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:42:06,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/4ae1cca7e774421e8189447feebf3861 is 50, key is test_row_0/A:col10/1731307326189/Put/seqid=0 2024-11-11T06:42:06,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742244_1420 (size=14341) 2024-11-11T06:42:06,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307386224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307386229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307386230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307386231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307386231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T06:42:06,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T06:42:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307386339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307386339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307386348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307386349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307386349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T06:42:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T06:42:06,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307386549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307386549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307386559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307386559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307386559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T06:42:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/4ae1cca7e774421e8189447feebf3861 2024-11-11T06:42:06,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/96ac3b97f9da45a5bce86972d9648042 is 50, key is test_row_0/B:col10/1731307326189/Put/seqid=0 2024-11-11T06:42:06,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742245_1421 (size=12001) 2024-11-11T06:42:06,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T06:42:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T06:42:06,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307386859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307386859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307386869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307386869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307386869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:06,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T06:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:06,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:06,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:07,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/96ac3b97f9da45a5bce86972d9648042 2024-11-11T06:42:07,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/20725f1328604801a8ca7c6d17e741e2 is 50, key is test_row_0/C:col10/1731307326189/Put/seqid=0 2024-11-11T06:42:07,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742246_1422 (size=12001) 2024-11-11T06:42:07,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/20725f1328604801a8ca7c6d17e741e2 2024-11-11T06:42:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/4ae1cca7e774421e8189447feebf3861 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ae1cca7e774421e8189447feebf3861 2024-11-11T06:42:07,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ae1cca7e774421e8189447feebf3861, entries=200, sequenceid=13, filesize=14.0 K 2024-11-11T06:42:07,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/96ac3b97f9da45a5bce86972d9648042 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/96ac3b97f9da45a5bce86972d9648042 2024-11-11T06:42:07,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T06:42:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:07,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:07,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/96ac3b97f9da45a5bce86972d9648042, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T06:42:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/20725f1328604801a8ca7c6d17e741e2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20725f1328604801a8ca7c6d17e741e2 2024-11-11T06:42:07,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20725f1328604801a8ca7c6d17e741e2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T06:42:07,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for c363d62112ab347adf2457471c10bf81 in 896ms, sequenceid=13, compaction requested=false 2024-11-11T06:42:07,088 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T06:42:07,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:07,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T06:42:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:07,235 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:42:07,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/1362a477a5d04bdd86a9d57b1d3fc650 is 50, key is test_row_0/A:col10/1731307326222/Put/seqid=0 2024-11-11T06:42:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742247_1423 (size=12001) 2024-11-11T06:42:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T06:42:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:07,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307387411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307387411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307387419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307387419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307387420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307387529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307387529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307387529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307387539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307387539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,646 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/1362a477a5d04bdd86a9d57b1d3fc650 2024-11-11T06:42:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/2646bd104beb40dd83447b8fe7812ed6 is 50, key is test_row_0/B:col10/1731307326222/Put/seqid=0 2024-11-11T06:42:07,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742248_1424 (size=12001) 2024-11-11T06:42:07,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307387738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307387739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307387739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307387749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307387749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307388049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307388050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,055 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/2646bd104beb40dd83447b8fe7812ed6 2024-11-11T06:42:08,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307388050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/c3f05c81410e4240baa8a31ff4cc9aca is 50, key is test_row_0/C:col10/1731307326222/Put/seqid=0 2024-11-11T06:42:08,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742249_1425 (size=12001) 2024-11-11T06:42:08,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307388059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307388059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T06:42:08,464 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/c3f05c81410e4240baa8a31ff4cc9aca 2024-11-11T06:42:08,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/1362a477a5d04bdd86a9d57b1d3fc650 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/1362a477a5d04bdd86a9d57b1d3fc650 2024-11-11T06:42:08,473 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/1362a477a5d04bdd86a9d57b1d3fc650, entries=150, sequenceid=38, filesize=11.7 K 2024-11-11T06:42:08,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/2646bd104beb40dd83447b8fe7812ed6 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/2646bd104beb40dd83447b8fe7812ed6 2024-11-11T06:42:08,477 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/2646bd104beb40dd83447b8fe7812ed6, entries=150, sequenceid=38, filesize=11.7 K 2024-11-11T06:42:08,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/c3f05c81410e4240baa8a31ff4cc9aca as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c3f05c81410e4240baa8a31ff4cc9aca 2024-11-11T06:42:08,481 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c3f05c81410e4240baa8a31ff4cc9aca, entries=150, sequenceid=38, filesize=11.7 K 2024-11-11T06:42:08,482 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c363d62112ab347adf2457471c10bf81 in 1247ms, sequenceid=38, compaction requested=false 2024-11-11T06:42:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T06:42:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T06:42:08,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T06:42:08,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3180 sec 2024-11-11T06:42:08,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.3240 sec 2024-11-11T06:42:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:08,562 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:08,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:42:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/988fcd4a8793435aafc865b9f3025b89 is 50, key is test_row_0/A:col10/1731307328561/Put/seqid=0 2024-11-11T06:42:08,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742250_1426 (size=14341) 2024-11-11T06:42:08,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307388601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307388602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307388611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307388613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307388613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307388719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307388719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307388729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307388729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307388729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307388928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307388929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307388939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307388939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307388939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:08,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/988fcd4a8793435aafc865b9f3025b89 2024-11-11T06:42:08,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00f0d4747e4b48f5b4885fa881a39bad is 50, key is test_row_0/B:col10/1731307328561/Put/seqid=0 2024-11-11T06:42:08,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742251_1427 (size=12001) 2024-11-11T06:42:08,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00f0d4747e4b48f5b4885fa881a39bad 2024-11-11T06:42:08,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/763d32313f8548b39de1d370b6ecac32 is 50, key is test_row_0/C:col10/1731307328561/Put/seqid=0 2024-11-11T06:42:08,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742252_1428 (size=12001) 2024-11-11T06:42:08,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/763d32313f8548b39de1d370b6ecac32 2024-11-11T06:42:08,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/988fcd4a8793435aafc865b9f3025b89 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/988fcd4a8793435aafc865b9f3025b89 2024-11-11T06:42:09,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/988fcd4a8793435aafc865b9f3025b89, entries=200, sequenceid=50, filesize=14.0 K 2024-11-11T06:42:09,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00f0d4747e4b48f5b4885fa881a39bad as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00f0d4747e4b48f5b4885fa881a39bad 2024-11-11T06:42:09,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00f0d4747e4b48f5b4885fa881a39bad, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T06:42:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/763d32313f8548b39de1d370b6ecac32 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/763d32313f8548b39de1d370b6ecac32 2024-11-11T06:42:09,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/763d32313f8548b39de1d370b6ecac32, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T06:42:09,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for c363d62112ab347adf2457471c10bf81 in 447ms, sequenceid=50, compaction requested=true 2024-11-11T06:42:09,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:09,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:09,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:09,009 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:09,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:09,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:09,009 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:09,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:09,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/B is initiating minor compaction (all files) 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/A is initiating minor compaction (all files) 2024-11-11T06:42:09,010 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/A in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:09,010 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/B in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:09,010 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ae1cca7e774421e8189447feebf3861, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/1362a477a5d04bdd86a9d57b1d3fc650, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/988fcd4a8793435aafc865b9f3025b89] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=39.7 K 2024-11-11T06:42:09,010 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/96ac3b97f9da45a5bce86972d9648042, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/2646bd104beb40dd83447b8fe7812ed6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00f0d4747e4b48f5b4885fa881a39bad] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=35.2 K 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae1cca7e774421e8189447feebf3861, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731307326188 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ac3b97f9da45a5bce86972d9648042, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731307326188 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1362a477a5d04bdd86a9d57b1d3fc650, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731307326222 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2646bd104beb40dd83447b8fe7812ed6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731307326222 2024-11-11T06:42:09,010 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 988fcd4a8793435aafc865b9f3025b89, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307327410 2024-11-11T06:42:09,011 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f0d4747e4b48f5b4885fa881a39bad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307327410 2024-11-11T06:42:09,019 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#A#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:09,019 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/c9450357a76749f1ac363c7904bb96e1 is 50, key is test_row_0/A:col10/1731307328561/Put/seqid=0 2024-11-11T06:42:09,021 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#B#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:09,021 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/b787cc2314b7468c8182184552eec86e is 50, key is test_row_0/B:col10/1731307328561/Put/seqid=0 2024-11-11T06:42:09,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742254_1430 (size=12104) 2024-11-11T06:42:09,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742253_1429 (size=12104) 2024-11-11T06:42:09,099 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T06:42:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:09,245 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:09,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T06:42:09,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/bc1ce35724ca46b4b334d6b5c68fe321 is 50, key is test_row_0/A:col10/1731307329245/Put/seqid=0 2024-11-11T06:42:09,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742255_1431 (size=16681) 2024-11-11T06:42:09,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307389254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307389255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307389257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307389258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307389259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307389368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307389369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307389369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307389369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307389369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,487 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/b787cc2314b7468c8182184552eec86e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b787cc2314b7468c8182184552eec86e 2024-11-11T06:42:09,487 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/c9450357a76749f1ac363c7904bb96e1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/c9450357a76749f1ac363c7904bb96e1 2024-11-11T06:42:09,490 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/A of c363d62112ab347adf2457471c10bf81 into c9450357a76749f1ac363c7904bb96e1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:09,490 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/B of c363d62112ab347adf2457471c10bf81 into b787cc2314b7468c8182184552eec86e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:09,491 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/A, priority=13, startTime=1731307329009; duration=0sec 2024-11-11T06:42:09,491 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/B, priority=13, startTime=1731307329009; duration=0sec 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:A 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:B 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:09,491 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/C is initiating minor compaction (all files) 2024-11-11T06:42:09,491 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/C in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:09,492 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20725f1328604801a8ca7c6d17e741e2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c3f05c81410e4240baa8a31ff4cc9aca, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/763d32313f8548b39de1d370b6ecac32] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=35.2 K 2024-11-11T06:42:09,492 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20725f1328604801a8ca7c6d17e741e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731307326188 2024-11-11T06:42:09,492 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f05c81410e4240baa8a31ff4cc9aca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731307326222 2024-11-11T06:42:09,492 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763d32313f8548b39de1d370b6ecac32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307327410 2024-11-11T06:42:09,497 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:09,497 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/7427e1aeb98e4f46bfa33d40c6da3c2b is 50, key is test_row_0/C:col10/1731307328561/Put/seqid=0 2024-11-11T06:42:09,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742256_1432 (size=12104) 2024-11-11T06:42:09,504 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/7427e1aeb98e4f46bfa33d40c6da3c2b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7427e1aeb98e4f46bfa33d40c6da3c2b 2024-11-11T06:42:09,507 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/C of c363d62112ab347adf2457471c10bf81 into 7427e1aeb98e4f46bfa33d40c6da3c2b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:09,507 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:09,507 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/C, priority=13, startTime=1731307329009; duration=0sec 2024-11-11T06:42:09,507 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:09,507 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:C 2024-11-11T06:42:09,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307389578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307389579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307389579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307389579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307389579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/bc1ce35724ca46b4b334d6b5c68fe321 2024-11-11T06:42:09,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/24fbdfec76b64833b5346ab647ca4b8d is 50, key is test_row_0/B:col10/1731307329245/Put/seqid=0 2024-11-11T06:42:09,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742257_1433 (size=12001) 2024-11-11T06:42:09,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/24fbdfec76b64833b5346ab647ca4b8d 2024-11-11T06:42:09,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/a7f96e6ee8ce41c28d64152f28a1328a is 50, key is test_row_0/C:col10/1731307329245/Put/seqid=0 2024-11-11T06:42:09,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742258_1434 (size=12001) 2024-11-11T06:42:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307389889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307389889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307389889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307389890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307389890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/a7f96e6ee8ce41c28d64152f28a1328a 2024-11-11T06:42:10,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/bc1ce35724ca46b4b334d6b5c68fe321 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/bc1ce35724ca46b4b334d6b5c68fe321 2024-11-11T06:42:10,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/bc1ce35724ca46b4b334d6b5c68fe321, entries=250, sequenceid=77, filesize=16.3 K 2024-11-11T06:42:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/24fbdfec76b64833b5346ab647ca4b8d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/24fbdfec76b64833b5346ab647ca4b8d 2024-11-11T06:42:10,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/24fbdfec76b64833b5346ab647ca4b8d, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T06:42:10,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/a7f96e6ee8ce41c28d64152f28a1328a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/a7f96e6ee8ce41c28d64152f28a1328a 2024-11-11T06:42:10,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/a7f96e6ee8ce41c28d64152f28a1328a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T06:42:10,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for c363d62112ab347adf2457471c10bf81 in 839ms, sequenceid=77, compaction requested=false 2024-11-11T06:42:10,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T06:42:10,329 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:10,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T06:42:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T06:42:10,331 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:10,332 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:10,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:10,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:10,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:42:10,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/b2cf7606c12e46c28ffa2d96ab7a8961 is 50, key is test_row_1/A:col10/1731307330402/Put/seqid=0 2024-11-11T06:42:10,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742259_1435 (size=14337) 2024-11-11T06:42:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T06:42:10,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307390457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307390461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307390466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307390467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307390468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:10,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307390579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307390579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307390579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307390579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307390579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:10,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:10,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:10,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:10,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T06:42:10,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:10,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:10,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:10,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:10,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307390789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307390789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307390789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307390789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307390789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:10,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/b2cf7606c12e46c28ffa2d96ab7a8961 2024-11-11T06:42:10,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/dcce2f50e9f14124875abbb7c8782f5b is 50, key is test_row_1/B:col10/1731307330402/Put/seqid=0 2024-11-11T06:42:10,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742260_1436 (size=9657) 2024-11-11T06:42:10,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:10,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:10,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:10,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:10,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T06:42:11,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307391099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307391099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307391099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307391099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307391109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/dcce2f50e9f14124875abbb7c8782f5b 2024-11-11T06:42:11,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/150ba75954bc4f7b94906cd6b9a297f3 is 50, key is test_row_1/C:col10/1731307330402/Put/seqid=0 2024-11-11T06:42:11,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742261_1437 (size=9657) 2024-11-11T06:42:11,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T06:42:11,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:11,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:11,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:11,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307391609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307391619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307391619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307391619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307391628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:11,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/150ba75954bc4f7b94906cd6b9a297f3 2024-11-11T06:42:11,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/b2cf7606c12e46c28ffa2d96ab7a8961 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/b2cf7606c12e46c28ffa2d96ab7a8961 2024-11-11T06:42:11,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/b2cf7606c12e46c28ffa2d96ab7a8961, entries=200, sequenceid=91, filesize=14.0 K 2024-11-11T06:42:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/dcce2f50e9f14124875abbb7c8782f5b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/dcce2f50e9f14124875abbb7c8782f5b 2024-11-11T06:42:11,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/dcce2f50e9f14124875abbb7c8782f5b, entries=100, sequenceid=91, filesize=9.4 K 2024-11-11T06:42:11,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/150ba75954bc4f7b94906cd6b9a297f3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/150ba75954bc4f7b94906cd6b9a297f3 2024-11-11T06:42:11,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/150ba75954bc4f7b94906cd6b9a297f3, entries=100, sequenceid=91, filesize=9.4 K 2024-11-11T06:42:11,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c363d62112ab347adf2457471c10bf81 in 1252ms, sequenceid=91, compaction requested=true 2024-11-11T06:42:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:11,659 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:11,659 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:11,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:11,660 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:11,660 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43122 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:11,660 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/A is initiating minor compaction (all files) 2024-11-11T06:42:11,660 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/B is initiating minor compaction (all files) 2024-11-11T06:42:11,660 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/A in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,660 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/B in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,660 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b787cc2314b7468c8182184552eec86e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/24fbdfec76b64833b5346ab647ca4b8d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/dcce2f50e9f14124875abbb7c8782f5b] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=33.0 K 2024-11-11T06:42:11,660 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/c9450357a76749f1ac363c7904bb96e1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/bc1ce35724ca46b4b334d6b5c68fe321, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/b2cf7606c12e46c28ffa2d96ab7a8961] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=42.1 K 2024-11-11T06:42:11,660 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b787cc2314b7468c8182184552eec86e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307327410 2024-11-11T06:42:11,660 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9450357a76749f1ac363c7904bb96e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307327410 2024-11-11T06:42:11,661 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 24fbdfec76b64833b5346ab647ca4b8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731307328610 2024-11-11T06:42:11,661 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc1ce35724ca46b4b334d6b5c68fe321, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731307328610 2024-11-11T06:42:11,661 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting dcce2f50e9f14124875abbb7c8782f5b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731307330401 2024-11-11T06:42:11,661 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2cf7606c12e46c28ffa2d96ab7a8961, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731307329253 2024-11-11T06:42:11,668 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:11,668 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/b92b4a22649144cdac1ebc5f2fdfbc37 is 50, key is test_row_0/B:col10/1731307329245/Put/seqid=0 2024-11-11T06:42:11,678 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#A#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:11,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742262_1438 (size=12207) 2024-11-11T06:42:11,679 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/47d63adf72c5439e89ed9b08cc059f85 is 50, key is test_row_0/A:col10/1731307329245/Put/seqid=0 2024-11-11T06:42:11,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742263_1439 (size=12207) 2024-11-11T06:42:11,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T06:42:11,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:11,705 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:42:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/6b38c2e2170a42cbaa7471eb75b94d19 is 50, key is test_row_0/A:col10/1731307330455/Put/seqid=0 2024-11-11T06:42:11,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742264_1440 (size=12001) 2024-11-11T06:42:11,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T06:42:12,082 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/b92b4a22649144cdac1ebc5f2fdfbc37 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b92b4a22649144cdac1ebc5f2fdfbc37 2024-11-11T06:42:12,085 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/47d63adf72c5439e89ed9b08cc059f85 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/47d63adf72c5439e89ed9b08cc059f85 2024-11-11T06:42:12,086 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/B of c363d62112ab347adf2457471c10bf81 into b92b4a22649144cdac1ebc5f2fdfbc37(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:12,086 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:12,086 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/B, priority=13, startTime=1731307331659; duration=0sec 2024-11-11T06:42:12,087 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:12,087 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:B 2024-11-11T06:42:12,087 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:12,087 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:12,087 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/C is initiating minor compaction (all files) 2024-11-11T06:42:12,088 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/C in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:12,088 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7427e1aeb98e4f46bfa33d40c6da3c2b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/a7f96e6ee8ce41c28d64152f28a1328a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/150ba75954bc4f7b94906cd6b9a297f3] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=33.0 K 2024-11-11T06:42:12,088 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7427e1aeb98e4f46bfa33d40c6da3c2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731307327410 2024-11-11T06:42:12,088 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f96e6ee8ce41c28d64152f28a1328a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731307328610 2024-11-11T06:42:12,089 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 150ba75954bc4f7b94906cd6b9a297f3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731307330401 2024-11-11T06:42:12,090 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/A of c363d62112ab347adf2457471c10bf81 into 47d63adf72c5439e89ed9b08cc059f85(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:12,090 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:12,090 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/A, priority=13, startTime=1731307331659; duration=0sec 2024-11-11T06:42:12,090 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:12,090 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:A 2024-11-11T06:42:12,094 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#C#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:12,095 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/69b865568f434c11a83c42d702b43906 is 50, key is test_row_0/C:col10/1731307329245/Put/seqid=0 2024-11-11T06:42:12,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742265_1441 (size=12207) 2024-11-11T06:42:12,115 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/6b38c2e2170a42cbaa7471eb75b94d19 2024-11-11T06:42:12,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/7b691eeb834c4868befebdb4d230da27 is 50, key is test_row_0/B:col10/1731307330455/Put/seqid=0 2024-11-11T06:42:12,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742266_1442 (size=12001) 2024-11-11T06:42:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T06:42:12,501 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/69b865568f434c11a83c42d702b43906 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/69b865568f434c11a83c42d702b43906 2024-11-11T06:42:12,504 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/C of c363d62112ab347adf2457471c10bf81 into 69b865568f434c11a83c42d702b43906(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:12,504 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:12,504 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/C, priority=13, startTime=1731307331659; duration=0sec 2024-11-11T06:42:12,504 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:12,504 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:C 2024-11-11T06:42:12,524 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/7b691eeb834c4868befebdb4d230da27 2024-11-11T06:42:12,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ac46ac43b0ac4313929843116576ea47 is 50, key is test_row_0/C:col10/1731307330455/Put/seqid=0 2024-11-11T06:42:12,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742267_1443 (size=12001) 2024-11-11T06:42:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:12,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307392639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307392639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:12,645 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307392639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:12,645 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307392641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,645 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307392642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:12,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307392749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307392759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:12,934 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ac46ac43b0ac4313929843116576ea47 2024-11-11T06:42:12,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/6b38c2e2170a42cbaa7471eb75b94d19 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/6b38c2e2170a42cbaa7471eb75b94d19 2024-11-11T06:42:12,939 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/6b38c2e2170a42cbaa7471eb75b94d19, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T06:42:12,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/7b691eeb834c4868befebdb4d230da27 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b691eeb834c4868befebdb4d230da27 2024-11-11T06:42:12,942 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b691eeb834c4868befebdb4d230da27, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T06:42:12,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ac46ac43b0ac4313929843116576ea47 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac46ac43b0ac4313929843116576ea47 2024-11-11T06:42:12,945 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac46ac43b0ac4313929843116576ea47, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T06:42:12,946 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c363d62112ab347adf2457471c10bf81 in 1241ms, sequenceid=116, compaction requested=false 2024-11-11T06:42:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:12,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T06:42:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T06:42:12,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T06:42:12,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6140 sec 2024-11-11T06:42:12,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6200 sec 2024-11-11T06:42:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:12,962 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:12,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:42:12,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/d280b9dcdc364692800ca0a884099cac is 50, key is test_row_0/A:col10/1731307332961/Put/seqid=0 2024-11-11T06:42:12,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742268_1444 (size=12101) 2024-11-11T06:42:12,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/d280b9dcdc364692800ca0a884099cac 2024-11-11T06:42:12,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/c1c69fa252614915bcc964bb0d679a30 is 50, key is test_row_0/B:col10/1731307332961/Put/seqid=0 2024-11-11T06:42:12,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742269_1445 (size=12101) 2024-11-11T06:42:13,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307393040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307393041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:13,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307393159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307393159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:13,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307393369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:13,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307393369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:13,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/c1c69fa252614915bcc964bb0d679a30 2024-11-11T06:42:13,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6cc149320f394bf08db5a2004b2da4d2 is 50, key is test_row_0/C:col10/1731307332961/Put/seqid=0 2024-11-11T06:42:13,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742270_1446 (size=12101) 2024-11-11T06:42:13,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307393679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307393679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:13,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6cc149320f394bf08db5a2004b2da4d2 2024-11-11T06:42:13,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/d280b9dcdc364692800ca0a884099cac as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d280b9dcdc364692800ca0a884099cac 2024-11-11T06:42:13,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d280b9dcdc364692800ca0a884099cac, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T06:42:13,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/c1c69fa252614915bcc964bb0d679a30 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/c1c69fa252614915bcc964bb0d679a30 2024-11-11T06:42:13,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/c1c69fa252614915bcc964bb0d679a30, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T06:42:13,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6cc149320f394bf08db5a2004b2da4d2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6cc149320f394bf08db5a2004b2da4d2 2024-11-11T06:42:13,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6cc149320f394bf08db5a2004b2da4d2, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T06:42:13,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c363d62112ab347adf2457471c10bf81 in 842ms, sequenceid=131, compaction requested=true 2024-11-11T06:42:13,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:13,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:13,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:13,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:13,804 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:13,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:13,804 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:13,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:13,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:13,805 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:13,805 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:13,805 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/A is initiating minor compaction (all files) 2024-11-11T06:42:13,805 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/B is initiating minor compaction (all files) 2024-11-11T06:42:13,805 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/B in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:13,805 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/A in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:13,805 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/47d63adf72c5439e89ed9b08cc059f85, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/6b38c2e2170a42cbaa7471eb75b94d19, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d280b9dcdc364692800ca0a884099cac] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=35.5 K 2024-11-11T06:42:13,805 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b92b4a22649144cdac1ebc5f2fdfbc37, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b691eeb834c4868befebdb4d230da27, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/c1c69fa252614915bcc964bb0d679a30] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=35.5 K 2024-11-11T06:42:13,805 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b92b4a22649144cdac1ebc5f2fdfbc37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731307329245 2024-11-11T06:42:13,805 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d63adf72c5439e89ed9b08cc059f85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731307329245 2024-11-11T06:42:13,806 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b691eeb834c4868befebdb4d230da27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731307330455 2024-11-11T06:42:13,806 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b38c2e2170a42cbaa7471eb75b94d19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731307330455 2024-11-11T06:42:13,806 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c69fa252614915bcc964bb0d679a30, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307332634 2024-11-11T06:42:13,806 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting d280b9dcdc364692800ca0a884099cac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307332634 2024-11-11T06:42:13,813 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:13,814 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/d66945808e654a788fe1325ead3bed51 is 50, key is test_row_0/B:col10/1731307332961/Put/seqid=0 2024-11-11T06:42:13,820 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#A#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:13,821 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/35ccd7c8902f404d9c2a80fbcd19bec1 is 50, key is test_row_0/A:col10/1731307332961/Put/seqid=0 2024-11-11T06:42:13,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742271_1447 (size=12409) 2024-11-11T06:42:13,828 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/d66945808e654a788fe1325ead3bed51 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/d66945808e654a788fe1325ead3bed51 2024-11-11T06:42:13,835 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/B of c363d62112ab347adf2457471c10bf81 into d66945808e654a788fe1325ead3bed51(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:13,835 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:13,835 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/B, priority=13, startTime=1731307333804; duration=0sec 2024-11-11T06:42:13,835 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:13,835 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:B 2024-11-11T06:42:13,835 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:13,835 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:13,836 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/C is initiating minor compaction (all files) 2024-11-11T06:42:13,836 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/C in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:13,836 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/69b865568f434c11a83c42d702b43906, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac46ac43b0ac4313929843116576ea47, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6cc149320f394bf08db5a2004b2da4d2] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=35.5 K 2024-11-11T06:42:13,836 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b865568f434c11a83c42d702b43906, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731307329245 2024-11-11T06:42:13,836 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ac46ac43b0ac4313929843116576ea47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731307330455 2024-11-11T06:42:13,837 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc149320f394bf08db5a2004b2da4d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307332634 2024-11-11T06:42:13,842 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:13,842 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/5aaacc08906a4443b1f8d64ffe925f4c is 50, key is test_row_0/C:col10/1731307332961/Put/seqid=0 2024-11-11T06:42:13,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742273_1449 (size=12409) 2024-11-11T06:42:13,858 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/5aaacc08906a4443b1f8d64ffe925f4c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/5aaacc08906a4443b1f8d64ffe925f4c 2024-11-11T06:42:13,862 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/C of c363d62112ab347adf2457471c10bf81 into 5aaacc08906a4443b1f8d64ffe925f4c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:13,862 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:13,862 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/C, priority=13, startTime=1731307333804; duration=0sec 2024-11-11T06:42:13,862 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:13,862 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:C 2024-11-11T06:42:13,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742272_1448 (size=12409) 2024-11-11T06:42:13,872 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/35ccd7c8902f404d9c2a80fbcd19bec1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/35ccd7c8902f404d9c2a80fbcd19bec1 2024-11-11T06:42:13,876 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/A of c363d62112ab347adf2457471c10bf81 into 35ccd7c8902f404d9c2a80fbcd19bec1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:13,876 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:13,876 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/A, priority=13, startTime=1731307333804; duration=0sec 2024-11-11T06:42:13,876 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:13,877 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:A 2024-11-11T06:42:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:14,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:14,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:42:14,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/e859fda92a9b469ab2ce761bd7c78977 is 50, key is test_row_0/A:col10/1731307333037/Put/seqid=0 2024-11-11T06:42:14,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742274_1450 (size=14541) 2024-11-11T06:42:14,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307394213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307394217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307394319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307394329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T06:42:14,500 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:14,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T06:42:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T06:42:14,501 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:14,502 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:14,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:14,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307394529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307394539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/e859fda92a9b469ab2ce761bd7c78977 2024-11-11T06:42:14,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/7b005f3117284b8db76e5e31401faccc is 50, key is test_row_0/B:col10/1731307333037/Put/seqid=0 2024-11-11T06:42:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T06:42:14,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742275_1451 (size=12151) 2024-11-11T06:42:14,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T06:42:14,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:14,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:14,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:14,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:14,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:14,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307394659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307394659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307394659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,664 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:14,664 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,664 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T06:42:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:14,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T06:42:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307394839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307394849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:14,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T06:42:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:14,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/7b005f3117284b8db76e5e31401faccc 2024-11-11T06:42:15,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/3588a3a3b32c4dc0a2ff5c95225c45b5 is 50, key is test_row_0/C:col10/1731307333037/Put/seqid=0 2024-11-11T06:42:15,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742276_1452 (size=12151) 2024-11-11T06:42:15,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T06:42:15,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:15,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:15,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:15,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T06:42:15,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T06:42:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:15,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307395359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307395369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:15,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T06:42:15,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:15,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:15,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:15,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/3588a3a3b32c4dc0a2ff5c95225c45b5 2024-11-11T06:42:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:15,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/e859fda92a9b469ab2ce761bd7c78977 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e859fda92a9b469ab2ce761bd7c78977 2024-11-11T06:42:15,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e859fda92a9b469ab2ce761bd7c78977, entries=200, sequenceid=159, filesize=14.2 K 2024-11-11T06:42:15,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/7b005f3117284b8db76e5e31401faccc as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b005f3117284b8db76e5e31401faccc 2024-11-11T06:42:15,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b005f3117284b8db76e5e31401faccc, entries=150, sequenceid=159, filesize=11.9 K 2024-11-11T06:42:15,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/3588a3a3b32c4dc0a2ff5c95225c45b5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3588a3a3b32c4dc0a2ff5c95225c45b5 2024-11-11T06:42:15,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3588a3a3b32c4dc0a2ff5c95225c45b5, entries=150, sequenceid=159, filesize=11.9 K 2024-11-11T06:42:15,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c363d62112ab347adf2457471c10bf81 in 1245ms, sequenceid=159, compaction requested=false 2024-11-11T06:42:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:15,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T06:42:15,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:15,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T06:42:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/3c369db3a8cd4c958aaa6993d319171f is 50, key is test_row_0/A:col10/1731307334197/Put/seqid=0 2024-11-11T06:42:15,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742277_1453 (size=12151) 2024-11-11T06:42:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T06:42:15,982 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/3c369db3a8cd4c958aaa6993d319171f 2024-11-11T06:42:15,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/e3492f2eb50c4153a72afa99fdf30427 is 50, key is test_row_0/B:col10/1731307334197/Put/seqid=0 2024-11-11T06:42:15,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742278_1454 (size=12151) 2024-11-11T06:42:15,991 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/e3492f2eb50c4153a72afa99fdf30427 2024-11-11T06:42:15,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/4962fa484a8e4e44be2cc57a14f2c4da is 50, key is test_row_0/C:col10/1731307334197/Put/seqid=0 2024-11-11T06:42:15,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742279_1455 (size=12151) 2024-11-11T06:42:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:16,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:16,399 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/4962fa484a8e4e44be2cc57a14f2c4da 2024-11-11T06:42:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/3c369db3a8cd4c958aaa6993d319171f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/3c369db3a8cd4c958aaa6993d319171f 2024-11-11T06:42:16,405 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/3c369db3a8cd4c958aaa6993d319171f, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T06:42:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/e3492f2eb50c4153a72afa99fdf30427 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/e3492f2eb50c4153a72afa99fdf30427 2024-11-11T06:42:16,409 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/e3492f2eb50c4153a72afa99fdf30427, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T06:42:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/4962fa484a8e4e44be2cc57a14f2c4da as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/4962fa484a8e4e44be2cc57a14f2c4da 2024-11-11T06:42:16,412 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/4962fa484a8e4e44be2cc57a14f2c4da, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T06:42:16,413 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=53.67 KB/54960 for c363d62112ab347adf2457471c10bf81 in 838ms, sequenceid=170, compaction requested=true 2024-11-11T06:42:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T06:42:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T06:42:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:16,414 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:16,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:42:16,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T06:42:16,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9120 sec 2024-11-11T06:42:16,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/0cb9fa9757e54e53a10c75f624f703a2 is 50, key is test_row_0/A:col10/1731307336394/Put/seqid=0 2024-11-11T06:42:16,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.9170 sec 2024-11-11T06:42:16,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742280_1456 (size=19321) 2024-11-11T06:42:16,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307396479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:16,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307396482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307396589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:16,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307396599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T06:42:16,649 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:16,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T06:42:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T06:42:16,651 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:16,651 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:16,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T06:42:16,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T06:42:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:16,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:16,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:16,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307396799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307396809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:16,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/0cb9fa9757e54e53a10c75f624f703a2 2024-11-11T06:42:16,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/cf71a74d6c1d44a1916dd56bff64fbc1 is 50, key is test_row_0/B:col10/1731307336394/Put/seqid=0 2024-11-11T06:42:16,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742281_1457 (size=12151) 2024-11-11T06:42:16,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T06:42:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:16,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T06:42:17,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T06:42:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307397119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:17,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307397119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:17,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/cf71a74d6c1d44a1916dd56bff64fbc1 2024-11-11T06:42:17,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/3555ca9c5e2940a08676c1d07b22a5d3 is 50, key is test_row_0/C:col10/1731307336394/Put/seqid=0 2024-11-11T06:42:17,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742282_1458 (size=12151) 2024-11-11T06:42:17,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T06:42:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T06:42:17,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T06:42:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T06:42:17,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:17,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:17,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/3555ca9c5e2940a08676c1d07b22a5d3 2024-11-11T06:42:17,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/0cb9fa9757e54e53a10c75f624f703a2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0cb9fa9757e54e53a10c75f624f703a2 2024-11-11T06:42:17,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307397639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:17,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307397639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:17,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0cb9fa9757e54e53a10c75f624f703a2, entries=300, sequenceid=183, filesize=18.9 K 2024-11-11T06:42:17,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/cf71a74d6c1d44a1916dd56bff64fbc1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/cf71a74d6c1d44a1916dd56bff64fbc1 2024-11-11T06:42:17,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/cf71a74d6c1d44a1916dd56bff64fbc1, entries=150, sequenceid=183, filesize=11.9 K 2024-11-11T06:42:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/3555ca9c5e2940a08676c1d07b22a5d3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3555ca9c5e2940a08676c1d07b22a5d3 2024-11-11T06:42:17,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3555ca9c5e2940a08676c1d07b22a5d3, entries=150, sequenceid=183, filesize=11.9 K 2024-11-11T06:42:17,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c363d62112ab347adf2457471c10bf81 in 1242ms, sequenceid=183, compaction requested=true 2024-11-11T06:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:17,656 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:17,656 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58422 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/B is initiating minor compaction (all files) 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/A is initiating minor compaction (all files) 2024-11-11T06:42:17,657 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/A in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,657 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/B in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,657 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/d66945808e654a788fe1325ead3bed51, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b005f3117284b8db76e5e31401faccc, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/e3492f2eb50c4153a72afa99fdf30427, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/cf71a74d6c1d44a1916dd56bff64fbc1] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=47.7 K 2024-11-11T06:42:17,657 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/35ccd7c8902f404d9c2a80fbcd19bec1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e859fda92a9b469ab2ce761bd7c78977, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/3c369db3a8cd4c958aaa6993d319171f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0cb9fa9757e54e53a10c75f624f703a2] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=57.1 K 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting d66945808e654a788fe1325ead3bed51, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307332634 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ccd7c8902f404d9c2a80fbcd19bec1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307332634 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b005f3117284b8db76e5e31401faccc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731307333025 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting e859fda92a9b469ab2ce761bd7c78977, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731307333025 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting e3492f2eb50c4153a72afa99fdf30427, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731307334197 2024-11-11T06:42:17,657 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c369db3a8cd4c958aaa6993d319171f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731307334197 2024-11-11T06:42:17,658 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting cf71a74d6c1d44a1916dd56bff64fbc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307336394 2024-11-11T06:42:17,658 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cb9fa9757e54e53a10c75f624f703a2, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307336392 2024-11-11T06:42:17,680 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:17,680 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/4ace47890ae9401ca4e318648a084f4a is 50, key is test_row_0/B:col10/1731307336394/Put/seqid=0 2024-11-11T06:42:17,681 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#A#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:17,681 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/788b5a06184d49689a27ddb911d0f325 is 50, key is test_row_0/A:col10/1731307336394/Put/seqid=0 2024-11-11T06:42:17,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742283_1459 (size=12595) 2024-11-11T06:42:17,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742284_1460 (size=12595) 2024-11-11T06:42:17,694 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/788b5a06184d49689a27ddb911d0f325 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/788b5a06184d49689a27ddb911d0f325 2024-11-11T06:42:17,698 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c363d62112ab347adf2457471c10bf81/A of c363d62112ab347adf2457471c10bf81 into 788b5a06184d49689a27ddb911d0f325(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:17,698 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:17,698 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/A, priority=12, startTime=1731307337656; duration=0sec 2024-11-11T06:42:17,698 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:17,698 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:A 2024-11-11T06:42:17,698 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:42:17,699 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:42:17,699 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/C is initiating minor compaction (all files) 2024-11-11T06:42:17,699 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/C in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,699 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/5aaacc08906a4443b1f8d64ffe925f4c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3588a3a3b32c4dc0a2ff5c95225c45b5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/4962fa484a8e4e44be2cc57a14f2c4da, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3555ca9c5e2940a08676c1d07b22a5d3] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=47.7 K 2024-11-11T06:42:17,699 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aaacc08906a4443b1f8d64ffe925f4c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307332634 2024-11-11T06:42:17,699 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3588a3a3b32c4dc0a2ff5c95225c45b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731307333025 2024-11-11T06:42:17,700 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4962fa484a8e4e44be2cc57a14f2c4da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731307334197 2024-11-11T06:42:17,700 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3555ca9c5e2940a08676c1d07b22a5d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307336394 2024-11-11T06:42:17,708 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:17,709 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ff045d461ba64983bca3f8970fc3bc1e is 50, key is test_row_0/C:col10/1731307336394/Put/seqid=0 2024-11-11T06:42:17,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742285_1461 (size=12595) 2024-11-11T06:42:17,717 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ff045d461ba64983bca3f8970fc3bc1e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ff045d461ba64983bca3f8970fc3bc1e 2024-11-11T06:42:17,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T06:42:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:17,719 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T06:42:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/7e32a104aef4426283d1989fd0aab32e is 50, key is test_row_0/A:col10/1731307336478/Put/seqid=0 2024-11-11T06:42:17,724 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c363d62112ab347adf2457471c10bf81/C of c363d62112ab347adf2457471c10bf81 into ff045d461ba64983bca3f8970fc3bc1e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:17,725 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:17,725 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/C, priority=12, startTime=1731307337656; duration=0sec 2024-11-11T06:42:17,725 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:17,725 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:C 2024-11-11T06:42:17,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742286_1462 (size=12151) 2024-11-11T06:42:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T06:42:18,088 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/4ace47890ae9401ca4e318648a084f4a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/4ace47890ae9401ca4e318648a084f4a 2024-11-11T06:42:18,092 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c363d62112ab347adf2457471c10bf81/B of c363d62112ab347adf2457471c10bf81 into 4ace47890ae9401ca4e318648a084f4a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:18,092 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:18,092 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/B, priority=12, startTime=1731307337656; duration=0sec 2024-11-11T06:42:18,092 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:18,092 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:B 2024-11-11T06:42:18,132 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/7e32a104aef4426283d1989fd0aab32e 2024-11-11T06:42:18,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/109aaf47afe64e358df5ce90e4d3f5d1 is 50, key is test_row_0/B:col10/1731307336478/Put/seqid=0 2024-11-11T06:42:18,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742287_1463 (size=12151) 2024-11-11T06:42:18,543 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/109aaf47afe64e358df5ce90e4d3f5d1 2024-11-11T06:42:18,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ef786cfd4c924aaa83f7a41069bb7af8 is 50, key is test_row_0/C:col10/1731307336478/Put/seqid=0 2024-11-11T06:42:18,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742288_1464 (size=12151) 2024-11-11T06:42:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:18,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:18,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47798 deadline: 1731307398679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:18,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307398681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:18,685 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:18,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307398682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:18,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47840 deadline: 1731307398689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:18,693 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47852 deadline: 1731307398699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:18,700 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_1' in c363d62112ab347adf2457471c10bf81 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:18,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307398789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T06:42:18,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307398799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:18,953 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ef786cfd4c924aaa83f7a41069bb7af8 2024-11-11T06:42:18,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/7e32a104aef4426283d1989fd0aab32e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/7e32a104aef4426283d1989fd0aab32e 2024-11-11T06:42:18,959 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/7e32a104aef4426283d1989fd0aab32e, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T06:42:18,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/109aaf47afe64e358df5ce90e4d3f5d1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/109aaf47afe64e358df5ce90e4d3f5d1 2024-11-11T06:42:18,962 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/109aaf47afe64e358df5ce90e4d3f5d1, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T06:42:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ef786cfd4c924aaa83f7a41069bb7af8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ef786cfd4c924aaa83f7a41069bb7af8 2024-11-11T06:42:18,966 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ef786cfd4c924aaa83f7a41069bb7af8, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T06:42:18,967 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c363d62112ab347adf2457471c10bf81 in 1248ms, sequenceid=207, compaction requested=false 2024-11-11T06:42:18,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:18,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:18,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T06:42:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T06:42:18,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T06:42:18,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3170 sec 2024-11-11T06:42:18,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.3210 sec 2024-11-11T06:42:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:19,003 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:19,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T06:42:19,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/55c4e84e20a146ee8abaead6d6565cc5 is 50, key is test_row_0/A:col10/1731307338680/Put/seqid=0 2024-11-11T06:42:19,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742289_1465 (size=14541) 2024-11-11T06:42:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307399058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307399064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:19,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307399169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:19,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307399169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:19,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307399379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:19,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:19,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307399379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:19,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/55c4e84e20a146ee8abaead6d6565cc5 2024-11-11T06:42:19,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/b016e63a8c454892bba85cfa30c4a86d is 50, key is test_row_0/B:col10/1731307338680/Put/seqid=0 2024-11-11T06:42:19,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742290_1466 (size=12151) 2024-11-11T06:42:19,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/b016e63a8c454892bba85cfa30c4a86d 2024-11-11T06:42:19,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/20ee23651fac4577a29d619def145cbf is 50, key is test_row_0/C:col10/1731307338680/Put/seqid=0 2024-11-11T06:42:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742291_1467 (size=12151) 2024-11-11T06:42:19,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307399689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:19,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307399699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:19,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/20ee23651fac4577a29d619def145cbf 2024-11-11T06:42:19,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/55c4e84e20a146ee8abaead6d6565cc5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/55c4e84e20a146ee8abaead6d6565cc5 2024-11-11T06:42:19,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/55c4e84e20a146ee8abaead6d6565cc5, entries=200, sequenceid=223, filesize=14.2 K 2024-11-11T06:42:19,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/b016e63a8c454892bba85cfa30c4a86d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b016e63a8c454892bba85cfa30c4a86d 2024-11-11T06:42:19,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b016e63a8c454892bba85cfa30c4a86d, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T06:42:19,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/20ee23651fac4577a29d619def145cbf as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20ee23651fac4577a29d619def145cbf 2024-11-11T06:42:19,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20ee23651fac4577a29d619def145cbf, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T06:42:19,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c363d62112ab347adf2457471c10bf81 in 879ms, sequenceid=223, compaction requested=true 2024-11-11T06:42:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:19,882 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:19,882 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:19,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/A is initiating minor compaction (all files) 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/B is initiating minor compaction (all files) 2024-11-11T06:42:19,883 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/B in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:19,883 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/A in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:19,883 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/788b5a06184d49689a27ddb911d0f325, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/7e32a104aef4426283d1989fd0aab32e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/55c4e84e20a146ee8abaead6d6565cc5] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=38.4 K 2024-11-11T06:42:19,883 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/4ace47890ae9401ca4e318648a084f4a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/109aaf47afe64e358df5ce90e4d3f5d1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b016e63a8c454892bba85cfa30c4a86d] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=36.0 K 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ace47890ae9401ca4e318648a084f4a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307336394 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788b5a06184d49689a27ddb911d0f325, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307336394 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 109aaf47afe64e358df5ce90e4d3f5d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307336467 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e32a104aef4426283d1989fd0aab32e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307336467 2024-11-11T06:42:19,883 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b016e63a8c454892bba85cfa30c4a86d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731307338676 2024-11-11T06:42:19,884 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c4e84e20a146ee8abaead6d6565cc5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731307338676 2024-11-11T06:42:19,889 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:19,889 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:19,889 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/0436ca50fce6408d98c7b2db8b77b4ec is 50, key is test_row_0/A:col10/1731307338680/Put/seqid=0 2024-11-11T06:42:19,889 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/887ea114f2e0479ba46822ca42fd4bff is 50, key is test_row_0/B:col10/1731307338680/Put/seqid=0 2024-11-11T06:42:19,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742292_1468 (size=12697) 2024-11-11T06:42:19,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742293_1469 (size=12697) 2024-11-11T06:42:19,895 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/887ea114f2e0479ba46822ca42fd4bff as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/887ea114f2e0479ba46822ca42fd4bff 2024-11-11T06:42:19,900 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/B of c363d62112ab347adf2457471c10bf81 into 887ea114f2e0479ba46822ca42fd4bff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:19,900 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:19,900 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/B, priority=13, startTime=1731307339882; duration=0sec 2024-11-11T06:42:19,900 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:19,900 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:B 2024-11-11T06:42:19,900 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:19,900 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:19,900 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/C is initiating minor compaction (all files) 2024-11-11T06:42:19,900 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/C in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:19,901 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ff045d461ba64983bca3f8970fc3bc1e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ef786cfd4c924aaa83f7a41069bb7af8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20ee23651fac4577a29d619def145cbf] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=36.0 K 2024-11-11T06:42:19,901 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ff045d461ba64983bca3f8970fc3bc1e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731307336394 2024-11-11T06:42:19,901 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ef786cfd4c924aaa83f7a41069bb7af8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731307336467 2024-11-11T06:42:19,901 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 20ee23651fac4577a29d619def145cbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731307338676 2024-11-11T06:42:19,905 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:19,906 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6d120d26882a43a9a3ee4c1c5259ba60 is 50, key is test_row_0/C:col10/1731307338680/Put/seqid=0 2024-11-11T06:42:19,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742294_1470 (size=12697) 2024-11-11T06:42:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:20,204 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:20,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T06:42:20,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/d17396a4879943acbc452bc8c312fdd3 is 50, key is test_row_0/A:col10/1731307339062/Put/seqid=0 2024-11-11T06:42:20,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742295_1471 (size=14541) 2024-11-11T06:42:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307400239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:20,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307400241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:20,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:20,297 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/0436ca50fce6408d98c7b2db8b77b4ec as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0436ca50fce6408d98c7b2db8b77b4ec 2024-11-11T06:42:20,301 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/A of c363d62112ab347adf2457471c10bf81 into 0436ca50fce6408d98c7b2db8b77b4ec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:20,301 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:20,301 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/A, priority=13, startTime=1731307339882; duration=0sec 2024-11-11T06:42:20,301 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:20,301 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:A 2024-11-11T06:42:20,312 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6d120d26882a43a9a3ee4c1c5259ba60 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6d120d26882a43a9a3ee4c1c5259ba60 2024-11-11T06:42:20,315 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/C of c363d62112ab347adf2457471c10bf81 into 6d120d26882a43a9a3ee4c1c5259ba60(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:20,315 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:20,316 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/C, priority=13, startTime=1731307339882; duration=0sec 2024-11-11T06:42:20,316 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:20,316 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:C 2024-11-11T06:42:20,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307400349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307400358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307400559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:20,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307400569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:20,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/d17396a4879943acbc452bc8c312fdd3 2024-11-11T06:42:20,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00ca0e27c5ad49b6a3844632dede78ee is 50, key is test_row_0/B:col10/1731307339062/Put/seqid=0 2024-11-11T06:42:20,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742296_1472 (size=12151) 2024-11-11T06:42:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T06:42:20,819 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:20,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T06:42:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T06:42:20,821 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:20,822 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:20,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307400868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:20,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307400879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T06:42:20,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T06:42:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:20,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00ca0e27c5ad49b6a3844632dede78ee 2024-11-11T06:42:21,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/0367331d68a24624985e246644a056c7 is 50, key is test_row_0/C:col10/1731307339062/Put/seqid=0 2024-11-11T06:42:21,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742297_1473 (size=12151) 2024-11-11T06:42:21,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T06:42:21,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:21,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:21,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:21,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T06:42:21,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T06:42:21,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:21,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:21,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:21,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307401378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:21,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307401389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:21,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/0367331d68a24624985e246644a056c7 2024-11-11T06:42:21,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/d17396a4879943acbc452bc8c312fdd3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d17396a4879943acbc452bc8c312fdd3 2024-11-11T06:42:21,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T06:42:21,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:21,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:21,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:21,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:21,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d17396a4879943acbc452bc8c312fdd3, entries=200, sequenceid=247, filesize=14.2 K 2024-11-11T06:42:21,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00ca0e27c5ad49b6a3844632dede78ee as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ca0e27c5ad49b6a3844632dede78ee 2024-11-11T06:42:21,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ca0e27c5ad49b6a3844632dede78ee, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T06:42:21,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/0367331d68a24624985e246644a056c7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0367331d68a24624985e246644a056c7 2024-11-11T06:42:21,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0367331d68a24624985e246644a056c7, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T06:42:21,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c363d62112ab347adf2457471c10bf81 in 1238ms, sequenceid=247, compaction requested=false 2024-11-11T06:42:21,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T06:42:21,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T06:42:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:21,584 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:42:21,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/ce3cd0f81b384efa8e46a74bb64502de is 50, key is test_row_0/A:col10/1731307340238/Put/seqid=0 2024-11-11T06:42:21,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742298_1474 (size=9857) 2024-11-11T06:42:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T06:42:21,991 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/ce3cd0f81b384efa8e46a74bb64502de 2024-11-11T06:42:21,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/23c865cd064844c69b7ba561f6958975 is 50, key is test_row_0/B:col10/1731307340238/Put/seqid=0 2024-11-11T06:42:21,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742299_1475 (size=9857) 2024-11-11T06:42:22,000 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/23c865cd064844c69b7ba561f6958975 2024-11-11T06:42:22,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ac9d5ce872254f7a879ae05464465314 is 50, key is test_row_0/C:col10/1731307340238/Put/seqid=0 2024-11-11T06:42:22,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742300_1476 (size=9857) 2024-11-11T06:42:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:22,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:22,416 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ac9d5ce872254f7a879ae05464465314 2024-11-11T06:42:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/ce3cd0f81b384efa8e46a74bb64502de as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/ce3cd0f81b384efa8e46a74bb64502de 2024-11-11T06:42:22,423 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/ce3cd0f81b384efa8e46a74bb64502de, entries=100, sequenceid=262, filesize=9.6 K 2024-11-11T06:42:22,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/23c865cd064844c69b7ba561f6958975 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/23c865cd064844c69b7ba561f6958975 2024-11-11T06:42:22,426 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/23c865cd064844c69b7ba561f6958975, entries=100, sequenceid=262, filesize=9.6 K 2024-11-11T06:42:22,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/ac9d5ce872254f7a879ae05464465314 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac9d5ce872254f7a879ae05464465314 2024-11-11T06:42:22,434 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac9d5ce872254f7a879ae05464465314, entries=100, sequenceid=262, filesize=9.6 K 2024-11-11T06:42:22,434 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=100.63 KB/103050 for c363d62112ab347adf2457471c10bf81 in 850ms, sequenceid=262, compaction requested=true 2024-11-11T06:42:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T06:42:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T06:42:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:22,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-11T06:42:22,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T06:42:22,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-11T06:42:22,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6180 sec 2024-11-11T06:42:22,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/183d2f824cc049beb4533060ce8e511f is 50, key is test_row_0/A:col10/1731307342435/Put/seqid=0 2024-11-11T06:42:22,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742301_1477 (size=17181) 2024-11-11T06:42:22,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307402491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:22,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307402495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:22,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:22,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307402609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307402609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307402818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307402819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:22,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/183d2f824cc049beb4533060ce8e511f 2024-11-11T06:42:22,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/1221937b7e0b467e8342eb8ba96b6d2e is 50, key is test_row_0/B:col10/1731307342435/Put/seqid=0 2024-11-11T06:42:22,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742302_1478 (size=12301) 2024-11-11T06:42:22,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/1221937b7e0b467e8342eb8ba96b6d2e 2024-11-11T06:42:22,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6412d197cf4041ecb505fda10ae57b00 is 50, key is test_row_0/C:col10/1731307342435/Put/seqid=0 2024-11-11T06:42:22,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742303_1479 (size=12301) 2024-11-11T06:42:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T06:42:22,969 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:22,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T06:42:22,971 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T06:42:22,971 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:22,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T06:42:23,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T06:42:23,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:23,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:23,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:23,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:23,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307403139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307403139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:23,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6412d197cf4041ecb505fda10ae57b00 2024-11-11T06:42:23,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/183d2f824cc049beb4533060ce8e511f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/183d2f824cc049beb4533060ce8e511f 2024-11-11T06:42:23,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/183d2f824cc049beb4533060ce8e511f, entries=250, sequenceid=281, filesize=16.8 K 2024-11-11T06:42:23,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/1221937b7e0b467e8342eb8ba96b6d2e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/1221937b7e0b467e8342eb8ba96b6d2e 2024-11-11T06:42:23,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T06:42:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:23,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:23,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/1221937b7e0b467e8342eb8ba96b6d2e, entries=150, sequenceid=281, filesize=12.0 K 2024-11-11T06:42:23,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6412d197cf4041ecb505fda10ae57b00 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6412d197cf4041ecb505fda10ae57b00 2024-11-11T06:42:23,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6412d197cf4041ecb505fda10ae57b00, entries=150, sequenceid=281, filesize=12.0 K 2024-11-11T06:42:23,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for c363d62112ab347adf2457471c10bf81 in 844ms, sequenceid=281, compaction requested=true 2024-11-11T06:42:23,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:23,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:23,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:23,281 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:42:23,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:23,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:23,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:23,281 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:42:23,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:23,281 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54276 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/A is initiating minor compaction (all files) 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/B is initiating minor compaction (all files) 2024-11-11T06:42:23,282 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/A in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:23,282 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/B in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:23,282 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0436ca50fce6408d98c7b2db8b77b4ec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d17396a4879943acbc452bc8c312fdd3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/ce3cd0f81b384efa8e46a74bb64502de, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/183d2f824cc049beb4533060ce8e511f] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=53.0 K 2024-11-11T06:42:23,282 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/887ea114f2e0479ba46822ca42fd4bff, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ca0e27c5ad49b6a3844632dede78ee, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/23c865cd064844c69b7ba561f6958975, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/1221937b7e0b467e8342eb8ba96b6d2e] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=45.9 K 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 887ea114f2e0479ba46822ca42fd4bff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731307338676 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0436ca50fce6408d98c7b2db8b77b4ec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731307338676 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ca0e27c5ad49b6a3844632dede78ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307339056 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting d17396a4879943acbc452bc8c312fdd3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307339056 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c865cd064844c69b7ba561f6958975, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731307340238 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce3cd0f81b384efa8e46a74bb64502de, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731307340238 2024-11-11T06:42:23,282 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 1221937b7e0b467e8342eb8ba96b6d2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731307342425 2024-11-11T06:42:23,283 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 183d2f824cc049beb4533060ce8e511f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731307342402 2024-11-11T06:42:23,288 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:23,288 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/e7720cee5dc14b72be61f48b31c1b71a is 50, key is test_row_0/A:col10/1731307342435/Put/seqid=0 2024-11-11T06:42:23,288 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:23,289 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/83f5dc6c851d45ba8a7fe28d7ed9e7e8 is 50, key is test_row_0/B:col10/1731307342435/Put/seqid=0 2024-11-11T06:42:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T06:42:23,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742305_1481 (size=12983) 2024-11-11T06:42:23,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742304_1480 (size=12983) 2024-11-11T06:42:23,295 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/e7720cee5dc14b72be61f48b31c1b71a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e7720cee5dc14b72be61f48b31c1b71a 2024-11-11T06:42:23,298 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c363d62112ab347adf2457471c10bf81/A of c363d62112ab347adf2457471c10bf81 into e7720cee5dc14b72be61f48b31c1b71a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:23,298 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:23,298 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/A, priority=12, startTime=1731307343281; duration=0sec 2024-11-11T06:42:23,298 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:23,298 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:A 2024-11-11T06:42:23,299 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T06:42:23,299 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T06:42:23,299 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/C is initiating minor compaction (all files) 2024-11-11T06:42:23,299 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/C in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:23,299 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6d120d26882a43a9a3ee4c1c5259ba60, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0367331d68a24624985e246644a056c7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac9d5ce872254f7a879ae05464465314, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6412d197cf4041ecb505fda10ae57b00] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=45.9 K 2024-11-11T06:42:23,300 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d120d26882a43a9a3ee4c1c5259ba60, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731307338676 2024-11-11T06:42:23,300 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0367331d68a24624985e246644a056c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731307339056 2024-11-11T06:42:23,300 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac9d5ce872254f7a879ae05464465314, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731307340238 2024-11-11T06:42:23,300 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6412d197cf4041ecb505fda10ae57b00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731307342425 2024-11-11T06:42:23,308 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:23,309 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/2362badf35a541cc91a0522f8300ab08 is 50, key is test_row_0/C:col10/1731307342435/Put/seqid=0 2024-11-11T06:42:23,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742306_1482 (size=12983) 2024-11-11T06:42:23,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T06:42:23,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:23,429 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T06:42:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/4ef1564b38b340c4b0ec3da393c590b8 is 50, key is test_row_0/A:col10/1731307342488/Put/seqid=0 2024-11-11T06:42:23,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742307_1483 (size=12301) 2024-11-11T06:42:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T06:42:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:23,695 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/83f5dc6c851d45ba8a7fe28d7ed9e7e8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/83f5dc6c851d45ba8a7fe28d7ed9e7e8 2024-11-11T06:42:23,698 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c363d62112ab347adf2457471c10bf81/B of c363d62112ab347adf2457471c10bf81 into 83f5dc6c851d45ba8a7fe28d7ed9e7e8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:23,698 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:23,698 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/B, priority=12, startTime=1731307343281; duration=0sec 2024-11-11T06:42:23,698 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:23,698 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:B 2024-11-11T06:42:23,716 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/2362badf35a541cc91a0522f8300ab08 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/2362badf35a541cc91a0522f8300ab08 2024-11-11T06:42:23,720 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c363d62112ab347adf2457471c10bf81/C of c363d62112ab347adf2457471c10bf81 into 2362badf35a541cc91a0522f8300ab08(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:23,720 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:23,720 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/C, priority=12, startTime=1731307343281; duration=0sec 2024-11-11T06:42:23,720 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:23,720 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:C 2024-11-11T06:42:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307403718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:23,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307403723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:23,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307403829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:23,836 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/4ef1564b38b340c4b0ec3da393c590b8 2024-11-11T06:42:23,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/f801e696e64b45549a00ff0d20216dd2 is 50, key is test_row_0/B:col10/1731307342488/Put/seqid=0 2024-11-11T06:42:23,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307403839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:23,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742308_1484 (size=12301) 2024-11-11T06:42:24,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307404039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:24,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307404049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T06:42:24,247 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/f801e696e64b45549a00ff0d20216dd2 2024-11-11T06:42:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/7131f7a1b1db4e9bb2dc640499094913 is 50, key is test_row_0/C:col10/1731307342488/Put/seqid=0 2024-11-11T06:42:24,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742309_1485 (size=12301) 2024-11-11T06:42:24,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307404349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:24,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307404359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:24,656 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/7131f7a1b1db4e9bb2dc640499094913 2024-11-11T06:42:24,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/4ef1564b38b340c4b0ec3da393c590b8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ef1564b38b340c4b0ec3da393c590b8 2024-11-11T06:42:24,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ef1564b38b340c4b0ec3da393c590b8, entries=150, sequenceid=299, filesize=12.0 K 2024-11-11T06:42:24,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/f801e696e64b45549a00ff0d20216dd2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/f801e696e64b45549a00ff0d20216dd2 2024-11-11T06:42:24,669 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/f801e696e64b45549a00ff0d20216dd2, entries=150, sequenceid=299, filesize=12.0 K 2024-11-11T06:42:24,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/7131f7a1b1db4e9bb2dc640499094913 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7131f7a1b1db4e9bb2dc640499094913 2024-11-11T06:42:24,673 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7131f7a1b1db4e9bb2dc640499094913, entries=150, sequenceid=299, filesize=12.0 K 2024-11-11T06:42:24,673 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for c363d62112ab347adf2457471c10bf81 in 1244ms, sequenceid=299, compaction requested=false 2024-11-11T06:42:24,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:24,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:24,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T06:42:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T06:42:24,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T06:42:24,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7040 sec 2024-11-11T06:42:24,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7070 sec 2024-11-11T06:42:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:24,864 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:24,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T06:42:24,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/dc06c63ae2b849eea396b8f85757a770 is 50, key is test_row_0/A:col10/1731307343711/Put/seqid=0 2024-11-11T06:42:24,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742310_1486 (size=14741) 2024-11-11T06:42:24,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307404928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:24,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307404933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:25,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307405049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:25,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307405049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T06:42:25,120 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:25,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-11T06:42:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T06:42:25,121 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:25,122 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:25,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T06:42:25,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307405258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:25,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307405259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:25,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T06:42:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:25,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/dc06c63ae2b849eea396b8f85757a770 2024-11-11T06:42:25,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00ed496a0f414809aaaf025a4e9977c4 is 50, key is test_row_0/B:col10/1731307343711/Put/seqid=0 2024-11-11T06:42:25,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742311_1487 (size=12301) 2024-11-11T06:42:25,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T06:42:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T06:42:25,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307405569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:25,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307405569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:25,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T06:42:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00ed496a0f414809aaaf025a4e9977c4 2024-11-11T06:42:25,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/372a998c4b7a434289e8a903256eabf3 is 50, key is test_row_0/C:col10/1731307343711/Put/seqid=0 2024-11-11T06:42:25,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742312_1488 (size=12301) 2024-11-11T06:42:25,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T06:42:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T06:42:25,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T06:42:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:25,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:26,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T06:42:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:26,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:26,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47828 deadline: 1731307406079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:26,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:26,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:26,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:26,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47822 deadline: 1731307406079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 , the old value is region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., hostname=ce2cf0244921,35807,1731307183669, seqNum=2 because the exception is null or not the one we care about 2024-11-11T06:42:26,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/372a998c4b7a434289e8a903256eabf3 2024-11-11T06:42:26,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/dc06c63ae2b849eea396b8f85757a770 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/dc06c63ae2b849eea396b8f85757a770 2024-11-11T06:42:26,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/dc06c63ae2b849eea396b8f85757a770, entries=200, sequenceid=321, filesize=14.4 K 2024-11-11T06:42:26,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/00ed496a0f414809aaaf025a4e9977c4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ed496a0f414809aaaf025a4e9977c4 2024-11-11T06:42:26,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ed496a0f414809aaaf025a4e9977c4, entries=150, sequenceid=321, filesize=12.0 K 2024-11-11T06:42:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/372a998c4b7a434289e8a903256eabf3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/372a998c4b7a434289e8a903256eabf3 2024-11-11T06:42:26,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/372a998c4b7a434289e8a903256eabf3, entries=150, sequenceid=321, filesize=12.0 K 2024-11-11T06:42:26,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for c363d62112ab347adf2457471c10bf81 in 1277ms, sequenceid=321, compaction requested=true 2024-11-11T06:42:26,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:26,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:26,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:26,142 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:26,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:26,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:26,142 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:26,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c363d62112ab347adf2457471c10bf81:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:26,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:26,142 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:26,142 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/A is initiating minor compaction (all files) 2024-11-11T06:42:26,142 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/A in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:26,142 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:26,143 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/B is initiating minor compaction (all files) 2024-11-11T06:42:26,143 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e7720cee5dc14b72be61f48b31c1b71a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ef1564b38b340c4b0ec3da393c590b8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/dc06c63ae2b849eea396b8f85757a770] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=39.1 K 2024-11-11T06:42:26,143 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/B in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:26,143 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/83f5dc6c851d45ba8a7fe28d7ed9e7e8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/f801e696e64b45549a00ff0d20216dd2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ed496a0f414809aaaf025a4e9977c4] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=36.7 K 2024-11-11T06:42:26,143 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7720cee5dc14b72be61f48b31c1b71a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731307342425 2024-11-11T06:42:26,143 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f5dc6c851d45ba8a7fe28d7ed9e7e8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731307342425 2024-11-11T06:42:26,143 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef1564b38b340c4b0ec3da393c590b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731307342488 2024-11-11T06:42:26,143 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting f801e696e64b45549a00ff0d20216dd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731307342488 2024-11-11T06:42:26,143 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc06c63ae2b849eea396b8f85757a770, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731307343700 2024-11-11T06:42:26,143 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ed496a0f414809aaaf025a4e9977c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731307343700 2024-11-11T06:42:26,150 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#B#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:26,150 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:26,150 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/127abb94117a4c1bbe888f08d29f3bb5 is 50, key is test_row_0/B:col10/1731307343711/Put/seqid=0 2024-11-11T06:42:26,150 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/64c3872df6354240845ee9f99d2b07ad is 50, key is test_row_0/A:col10/1731307343711/Put/seqid=0 2024-11-11T06:42:26,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742313_1489 (size=13085) 2024-11-11T06:42:26,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742314_1490 (size=13085) 2024-11-11T06:42:26,164 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-11-11T06:42:26,164 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:26,164 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,164 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:26,167 INFO [Thread-1915 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1915. 2024-11-11T06:42:26,167 DEBUG [Thread-1915 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:26,167 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,167 DEBUG [Thread-1915 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:26,169 INFO [Thread-1917 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1917. 2024-11-11T06:42:26,169 DEBUG [Thread-1917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:26,169 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,169 DEBUG [Thread-1917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:26,170 INFO [Thread-1913 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1913. 2024-11-11T06:42:26,170 DEBUG [Thread-1913 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:26,170 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,170 DEBUG [Thread-1913 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:26,173 INFO [Thread-1921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1921. 2024-11-11T06:42:26,173 DEBUG [Thread-1921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:26,173 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,173 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:26,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:26,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T06:42:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:26,189 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T06:42:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/9e77952b9f0e4fd28637f43b722535b3 is 50, key is test_row_0/A:col10/1731307344932/Put/seqid=0 2024-11-11T06:42:26,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742315_1491 (size=12301) 2024-11-11T06:42:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T06:42:26,562 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/127abb94117a4c1bbe888f08d29f3bb5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/127abb94117a4c1bbe888f08d29f3bb5 2024-11-11T06:42:26,563 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/64c3872df6354240845ee9f99d2b07ad as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/64c3872df6354240845ee9f99d2b07ad 2024-11-11T06:42:26,567 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/B of c363d62112ab347adf2457471c10bf81 into 127abb94117a4c1bbe888f08d29f3bb5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:26,567 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/A of c363d62112ab347adf2457471c10bf81 into 64c3872df6354240845ee9f99d2b07ad(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:26,567 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:26,567 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:26,567 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/A, priority=13, startTime=1731307346142; duration=0sec 2024-11-11T06:42:26,567 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/B, priority=13, startTime=1731307346142; duration=0sec 2024-11-11T06:42:26,568 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:26,568 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:B 2024-11-11T06:42:26,568 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:26,568 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:A 2024-11-11T06:42:26,568 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:26,569 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:26,569 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): c363d62112ab347adf2457471c10bf81/C is initiating minor compaction (all files) 2024-11-11T06:42:26,569 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c363d62112ab347adf2457471c10bf81/C in TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:26,569 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/2362badf35a541cc91a0522f8300ab08, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7131f7a1b1db4e9bb2dc640499094913, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/372a998c4b7a434289e8a903256eabf3] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp, totalSize=36.7 K 2024-11-11T06:42:26,569 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 2362badf35a541cc91a0522f8300ab08, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731307342425 2024-11-11T06:42:26,570 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 7131f7a1b1db4e9bb2dc640499094913, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731307342488 2024-11-11T06:42:26,570 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 372a998c4b7a434289e8a903256eabf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731307343700 2024-11-11T06:42:26,577 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c363d62112ab347adf2457471c10bf81#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:26,578 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6571d960445a4de685ad55056aef3bd9 is 50, key is test_row_0/C:col10/1731307343711/Put/seqid=0 2024-11-11T06:42:26,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742316_1492 (size=13085) 2024-11-11T06:42:26,597 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/9e77952b9f0e4fd28637f43b722535b3 2024-11-11T06:42:26,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/33a7587acad34beb8149549053b9ba31 is 50, key is test_row_0/B:col10/1731307344932/Put/seqid=0 2024-11-11T06:42:26,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742317_1493 (size=12301) 2024-11-11T06:42:26,994 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/6571d960445a4de685ad55056aef3bd9 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6571d960445a4de685ad55056aef3bd9 2024-11-11T06:42:27,000 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c363d62112ab347adf2457471c10bf81/C of c363d62112ab347adf2457471c10bf81 into 6571d960445a4de685ad55056aef3bd9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:27,001 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:27,001 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., storeName=c363d62112ab347adf2457471c10bf81/C, priority=13, startTime=1731307346142; duration=0sec 2024-11-11T06:42:27,001 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:27,001 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c363d62112ab347adf2457471c10bf81:C 2024-11-11T06:42:27,009 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/33a7587acad34beb8149549053b9ba31 2024-11-11T06:42:27,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/0bd43b5b014e4d47bf306bb6ad3e8806 is 50, key is test_row_0/C:col10/1731307344932/Put/seqid=0 2024-11-11T06:42:27,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742318_1494 (size=12301) 2024-11-11T06:42:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:27,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. as already flushing 2024-11-11T06:42:27,091 INFO [Thread-1902 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1902. 2024-11-11T06:42:27,091 DEBUG [Thread-1902 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:27,091 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:27,091 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:27,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:27,101 INFO [Thread-1908 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1908. 2024-11-11T06:42:27,101 DEBUG [Thread-1908 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:27,101 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:27,101 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:27,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T06:42:27,423 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/0bd43b5b014e4d47bf306bb6ad3e8806 2024-11-11T06:42:27,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/9e77952b9f0e4fd28637f43b722535b3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/9e77952b9f0e4fd28637f43b722535b3 2024-11-11T06:42:27,436 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/9e77952b9f0e4fd28637f43b722535b3, entries=150, sequenceid=337, filesize=12.0 K 2024-11-11T06:42:27,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/33a7587acad34beb8149549053b9ba31 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/33a7587acad34beb8149549053b9ba31 2024-11-11T06:42:27,442 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/33a7587acad34beb8149549053b9ba31, entries=150, sequenceid=337, filesize=12.0 K 2024-11-11T06:42:27,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/0bd43b5b014e4d47bf306bb6ad3e8806 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0bd43b5b014e4d47bf306bb6ad3e8806 2024-11-11T06:42:27,446 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0bd43b5b014e4d47bf306bb6ad3e8806, entries=150, sequenceid=337, filesize=12.0 K 2024-11-11T06:42:27,446 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=13.42 KB/13740 for c363d62112ab347adf2457471c10bf81 in 1257ms, sequenceid=337, compaction requested=false 2024-11-11T06:42:27,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for c363d62112ab347adf2457471c10bf81: 2024-11-11T06:42:27,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:27,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-11T06:42:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-11T06:42:27,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T06:42:27,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3260 sec 2024-11-11T06:42:27,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.3290 sec 2024-11-11T06:42:28,733 INFO [Thread-1904 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1904. 2024-11-11T06:42:28,733 DEBUG [Thread-1904 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:28,733 DEBUG [Thread-1904 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:28,734 DEBUG [Thread-1904 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:28,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:28,784 INFO [Thread-1906 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1906. 2024-11-11T06:42:28,784 INFO [Thread-1910 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1910. 2024-11-11T06:42:28,784 DEBUG [Thread-1906 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:28,784 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:28,784 DEBUG [Thread-1910 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:28,784 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:28,784 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:28,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:28,785 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:28,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T06:42:29,291 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1691 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5064 rows 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1682 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5043 rows 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1684 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5048 rows 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1678 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5033 rows 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1683 2024-11-11T06:42:29,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5047 rows 2024-11-11T06:42:29,292 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T06:42:29,292 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:42:29,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:29,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:29,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:29,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T06:42:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T06:42:29,298 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307349298"}]},"ts":"1731307349298"} 2024-11-11T06:42:29,300 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T06:42:29,300 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T06:42:29,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:42:29,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, UNASSIGN}] 2024-11-11T06:42:29,303 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, UNASSIGN 2024-11-11T06:42:29,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=c363d62112ab347adf2457471c10bf81, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:29,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, UNASSIGN because future has completed 2024-11-11T06:42:29,306 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:42:29,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:42:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T06:42:29,459 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:29,459 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:42:29,459 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing c363d62112ab347adf2457471c10bf81, disabling compactions & flushes 2024-11-11T06:42:29,459 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:29,460 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:29,460 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. after waiting 0 ms 2024-11-11T06:42:29,460 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:29,460 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing c363d62112ab347adf2457471c10bf81 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T06:42:29,469 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/f1151815026948de83218ee398a46e3d is 50, key is test_row_1/A:col10/1731307348781/Put/seqid=0 2024-11-11T06:42:29,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742319_1495 (size=9857) 2024-11-11T06:42:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T06:42:29,877 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/f1151815026948de83218ee398a46e3d 2024-11-11T06:42:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T06:42:29,933 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/fa6f9367504a4d24ac6c1c27b09483e6 is 50, key is test_row_1/B:col10/1731307348781/Put/seqid=0 2024-11-11T06:42:29,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742320_1496 (size=9857) 2024-11-11T06:42:30,338 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/fa6f9367504a4d24ac6c1c27b09483e6 2024-11-11T06:42:30,345 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/c7e14a8436054d908ba9598c5f0ae886 is 50, key is test_row_1/C:col10/1731307348781/Put/seqid=0 2024-11-11T06:42:30,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742321_1497 (size=9857) 2024-11-11T06:42:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T06:42:30,752 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/c7e14a8436054d908ba9598c5f0ae886 2024-11-11T06:42:30,762 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/A/f1151815026948de83218ee398a46e3d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/f1151815026948de83218ee398a46e3d 2024-11-11T06:42:30,766 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/f1151815026948de83218ee398a46e3d, entries=100, sequenceid=348, filesize=9.6 K 2024-11-11T06:42:30,767 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/B/fa6f9367504a4d24ac6c1c27b09483e6 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/fa6f9367504a4d24ac6c1c27b09483e6 2024-11-11T06:42:30,771 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/fa6f9367504a4d24ac6c1c27b09483e6, entries=100, sequenceid=348, filesize=9.6 K 2024-11-11T06:42:30,772 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/.tmp/C/c7e14a8436054d908ba9598c5f0ae886 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c7e14a8436054d908ba9598c5f0ae886 2024-11-11T06:42:30,775 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c7e14a8436054d908ba9598c5f0ae886, entries=100, sequenceid=348, filesize=9.6 K 2024-11-11T06:42:30,776 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c363d62112ab347adf2457471c10bf81 in 1316ms, sequenceid=348, compaction requested=true 2024-11-11T06:42:30,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ae1cca7e774421e8189447feebf3861, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/1362a477a5d04bdd86a9d57b1d3fc650, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/988fcd4a8793435aafc865b9f3025b89, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/c9450357a76749f1ac363c7904bb96e1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/bc1ce35724ca46b4b334d6b5c68fe321, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/b2cf7606c12e46c28ffa2d96ab7a8961, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/47d63adf72c5439e89ed9b08cc059f85, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/6b38c2e2170a42cbaa7471eb75b94d19, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/35ccd7c8902f404d9c2a80fbcd19bec1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d280b9dcdc364692800ca0a884099cac, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e859fda92a9b469ab2ce761bd7c78977, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/3c369db3a8cd4c958aaa6993d319171f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0cb9fa9757e54e53a10c75f624f703a2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/788b5a06184d49689a27ddb911d0f325, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/7e32a104aef4426283d1989fd0aab32e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/55c4e84e20a146ee8abaead6d6565cc5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0436ca50fce6408d98c7b2db8b77b4ec, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d17396a4879943acbc452bc8c312fdd3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/ce3cd0f81b384efa8e46a74bb64502de, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/183d2f824cc049beb4533060ce8e511f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e7720cee5dc14b72be61f48b31c1b71a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ef1564b38b340c4b0ec3da393c590b8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/dc06c63ae2b849eea396b8f85757a770] to archive 2024-11-11T06:42:30,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:30,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ae1cca7e774421e8189447feebf3861 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ae1cca7e774421e8189447feebf3861 2024-11-11T06:42:30,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/1362a477a5d04bdd86a9d57b1d3fc650 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/1362a477a5d04bdd86a9d57b1d3fc650 2024-11-11T06:42:30,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/988fcd4a8793435aafc865b9f3025b89 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/988fcd4a8793435aafc865b9f3025b89 2024-11-11T06:42:30,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/c9450357a76749f1ac363c7904bb96e1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/c9450357a76749f1ac363c7904bb96e1 2024-11-11T06:42:30,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/bc1ce35724ca46b4b334d6b5c68fe321 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/bc1ce35724ca46b4b334d6b5c68fe321 2024-11-11T06:42:30,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/b2cf7606c12e46c28ffa2d96ab7a8961 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/b2cf7606c12e46c28ffa2d96ab7a8961 2024-11-11T06:42:30,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/47d63adf72c5439e89ed9b08cc059f85 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/47d63adf72c5439e89ed9b08cc059f85 2024-11-11T06:42:30,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/6b38c2e2170a42cbaa7471eb75b94d19 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/6b38c2e2170a42cbaa7471eb75b94d19 2024-11-11T06:42:30,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/35ccd7c8902f404d9c2a80fbcd19bec1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/35ccd7c8902f404d9c2a80fbcd19bec1 2024-11-11T06:42:30,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d280b9dcdc364692800ca0a884099cac to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d280b9dcdc364692800ca0a884099cac 2024-11-11T06:42:30,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e859fda92a9b469ab2ce761bd7c78977 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e859fda92a9b469ab2ce761bd7c78977 2024-11-11T06:42:30,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/3c369db3a8cd4c958aaa6993d319171f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/3c369db3a8cd4c958aaa6993d319171f 2024-11-11T06:42:30,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0cb9fa9757e54e53a10c75f624f703a2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0cb9fa9757e54e53a10c75f624f703a2 2024-11-11T06:42:30,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/788b5a06184d49689a27ddb911d0f325 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/788b5a06184d49689a27ddb911d0f325 2024-11-11T06:42:30,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/7e32a104aef4426283d1989fd0aab32e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/7e32a104aef4426283d1989fd0aab32e 2024-11-11T06:42:30,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/55c4e84e20a146ee8abaead6d6565cc5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/55c4e84e20a146ee8abaead6d6565cc5 2024-11-11T06:42:30,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0436ca50fce6408d98c7b2db8b77b4ec to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/0436ca50fce6408d98c7b2db8b77b4ec 2024-11-11T06:42:30,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d17396a4879943acbc452bc8c312fdd3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/d17396a4879943acbc452bc8c312fdd3 2024-11-11T06:42:30,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/ce3cd0f81b384efa8e46a74bb64502de to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/ce3cd0f81b384efa8e46a74bb64502de 2024-11-11T06:42:30,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/183d2f824cc049beb4533060ce8e511f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/183d2f824cc049beb4533060ce8e511f 2024-11-11T06:42:30,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e7720cee5dc14b72be61f48b31c1b71a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/e7720cee5dc14b72be61f48b31c1b71a 2024-11-11T06:42:30,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ef1564b38b340c4b0ec3da393c590b8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/4ef1564b38b340c4b0ec3da393c590b8 2024-11-11T06:42:30,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/dc06c63ae2b849eea396b8f85757a770 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/dc06c63ae2b849eea396b8f85757a770 2024-11-11T06:42:30,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/96ac3b97f9da45a5bce86972d9648042, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/2646bd104beb40dd83447b8fe7812ed6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b787cc2314b7468c8182184552eec86e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00f0d4747e4b48f5b4885fa881a39bad, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/24fbdfec76b64833b5346ab647ca4b8d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b92b4a22649144cdac1ebc5f2fdfbc37, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/dcce2f50e9f14124875abbb7c8782f5b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b691eeb834c4868befebdb4d230da27, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/d66945808e654a788fe1325ead3bed51, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/c1c69fa252614915bcc964bb0d679a30, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b005f3117284b8db76e5e31401faccc, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/e3492f2eb50c4153a72afa99fdf30427, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/4ace47890ae9401ca4e318648a084f4a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/cf71a74d6c1d44a1916dd56bff64fbc1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/109aaf47afe64e358df5ce90e4d3f5d1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/887ea114f2e0479ba46822ca42fd4bff, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b016e63a8c454892bba85cfa30c4a86d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ca0e27c5ad49b6a3844632dede78ee, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/23c865cd064844c69b7ba561f6958975, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/83f5dc6c851d45ba8a7fe28d7ed9e7e8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/1221937b7e0b467e8342eb8ba96b6d2e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/f801e696e64b45549a00ff0d20216dd2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ed496a0f414809aaaf025a4e9977c4] to archive 2024-11-11T06:42:30,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:30,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/96ac3b97f9da45a5bce86972d9648042 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/96ac3b97f9da45a5bce86972d9648042 2024-11-11T06:42:30,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/2646bd104beb40dd83447b8fe7812ed6 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/2646bd104beb40dd83447b8fe7812ed6 2024-11-11T06:42:30,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b787cc2314b7468c8182184552eec86e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b787cc2314b7468c8182184552eec86e 2024-11-11T06:42:30,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00f0d4747e4b48f5b4885fa881a39bad to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00f0d4747e4b48f5b4885fa881a39bad 2024-11-11T06:42:30,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/24fbdfec76b64833b5346ab647ca4b8d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/24fbdfec76b64833b5346ab647ca4b8d 2024-11-11T06:42:30,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b92b4a22649144cdac1ebc5f2fdfbc37 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b92b4a22649144cdac1ebc5f2fdfbc37 2024-11-11T06:42:30,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/dcce2f50e9f14124875abbb7c8782f5b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/dcce2f50e9f14124875abbb7c8782f5b 2024-11-11T06:42:30,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b691eeb834c4868befebdb4d230da27 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b691eeb834c4868befebdb4d230da27 2024-11-11T06:42:30,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/d66945808e654a788fe1325ead3bed51 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/d66945808e654a788fe1325ead3bed51 2024-11-11T06:42:30,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/c1c69fa252614915bcc964bb0d679a30 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/c1c69fa252614915bcc964bb0d679a30 2024-11-11T06:42:30,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b005f3117284b8db76e5e31401faccc to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/7b005f3117284b8db76e5e31401faccc 2024-11-11T06:42:30,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/e3492f2eb50c4153a72afa99fdf30427 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/e3492f2eb50c4153a72afa99fdf30427 2024-11-11T06:42:30,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/4ace47890ae9401ca4e318648a084f4a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/4ace47890ae9401ca4e318648a084f4a 2024-11-11T06:42:30,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/cf71a74d6c1d44a1916dd56bff64fbc1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/cf71a74d6c1d44a1916dd56bff64fbc1 2024-11-11T06:42:30,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/109aaf47afe64e358df5ce90e4d3f5d1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/109aaf47afe64e358df5ce90e4d3f5d1 2024-11-11T06:42:30,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/887ea114f2e0479ba46822ca42fd4bff to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/887ea114f2e0479ba46822ca42fd4bff 2024-11-11T06:42:30,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b016e63a8c454892bba85cfa30c4a86d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/b016e63a8c454892bba85cfa30c4a86d 2024-11-11T06:42:30,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ca0e27c5ad49b6a3844632dede78ee to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ca0e27c5ad49b6a3844632dede78ee 2024-11-11T06:42:30,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/23c865cd064844c69b7ba561f6958975 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/23c865cd064844c69b7ba561f6958975 2024-11-11T06:42:30,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/83f5dc6c851d45ba8a7fe28d7ed9e7e8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/83f5dc6c851d45ba8a7fe28d7ed9e7e8 2024-11-11T06:42:30,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/1221937b7e0b467e8342eb8ba96b6d2e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/1221937b7e0b467e8342eb8ba96b6d2e 2024-11-11T06:42:30,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/f801e696e64b45549a00ff0d20216dd2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/f801e696e64b45549a00ff0d20216dd2 2024-11-11T06:42:30,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ed496a0f414809aaaf025a4e9977c4 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/00ed496a0f414809aaaf025a4e9977c4 2024-11-11T06:42:30,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20725f1328604801a8ca7c6d17e741e2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c3f05c81410e4240baa8a31ff4cc9aca, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7427e1aeb98e4f46bfa33d40c6da3c2b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/763d32313f8548b39de1d370b6ecac32, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/a7f96e6ee8ce41c28d64152f28a1328a, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/69b865568f434c11a83c42d702b43906, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/150ba75954bc4f7b94906cd6b9a297f3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac46ac43b0ac4313929843116576ea47, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/5aaacc08906a4443b1f8d64ffe925f4c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6cc149320f394bf08db5a2004b2da4d2, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3588a3a3b32c4dc0a2ff5c95225c45b5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/4962fa484a8e4e44be2cc57a14f2c4da, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ff045d461ba64983bca3f8970fc3bc1e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3555ca9c5e2940a08676c1d07b22a5d3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ef786cfd4c924aaa83f7a41069bb7af8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6d120d26882a43a9a3ee4c1c5259ba60, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20ee23651fac4577a29d619def145cbf, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0367331d68a24624985e246644a056c7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac9d5ce872254f7a879ae05464465314, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/2362badf35a541cc91a0522f8300ab08, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6412d197cf4041ecb505fda10ae57b00, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7131f7a1b1db4e9bb2dc640499094913, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/372a998c4b7a434289e8a903256eabf3] to archive 2024-11-11T06:42:30,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:30,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20725f1328604801a8ca7c6d17e741e2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20725f1328604801a8ca7c6d17e741e2 2024-11-11T06:42:30,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c3f05c81410e4240baa8a31ff4cc9aca to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c3f05c81410e4240baa8a31ff4cc9aca 2024-11-11T06:42:30,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7427e1aeb98e4f46bfa33d40c6da3c2b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7427e1aeb98e4f46bfa33d40c6da3c2b 2024-11-11T06:42:30,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/763d32313f8548b39de1d370b6ecac32 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/763d32313f8548b39de1d370b6ecac32 2024-11-11T06:42:30,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/a7f96e6ee8ce41c28d64152f28a1328a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/a7f96e6ee8ce41c28d64152f28a1328a 2024-11-11T06:42:30,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/69b865568f434c11a83c42d702b43906 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/69b865568f434c11a83c42d702b43906 2024-11-11T06:42:30,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/150ba75954bc4f7b94906cd6b9a297f3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/150ba75954bc4f7b94906cd6b9a297f3 2024-11-11T06:42:30,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac46ac43b0ac4313929843116576ea47 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac46ac43b0ac4313929843116576ea47 2024-11-11T06:42:30,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/5aaacc08906a4443b1f8d64ffe925f4c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/5aaacc08906a4443b1f8d64ffe925f4c 2024-11-11T06:42:30,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6cc149320f394bf08db5a2004b2da4d2 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6cc149320f394bf08db5a2004b2da4d2 2024-11-11T06:42:30,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3588a3a3b32c4dc0a2ff5c95225c45b5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3588a3a3b32c4dc0a2ff5c95225c45b5 2024-11-11T06:42:30,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/4962fa484a8e4e44be2cc57a14f2c4da to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/4962fa484a8e4e44be2cc57a14f2c4da 2024-11-11T06:42:30,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ff045d461ba64983bca3f8970fc3bc1e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ff045d461ba64983bca3f8970fc3bc1e 2024-11-11T06:42:30,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3555ca9c5e2940a08676c1d07b22a5d3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/3555ca9c5e2940a08676c1d07b22a5d3 2024-11-11T06:42:30,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ef786cfd4c924aaa83f7a41069bb7af8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ef786cfd4c924aaa83f7a41069bb7af8 2024-11-11T06:42:30,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6d120d26882a43a9a3ee4c1c5259ba60 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6d120d26882a43a9a3ee4c1c5259ba60 2024-11-11T06:42:30,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20ee23651fac4577a29d619def145cbf to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/20ee23651fac4577a29d619def145cbf 2024-11-11T06:42:30,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0367331d68a24624985e246644a056c7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0367331d68a24624985e246644a056c7 2024-11-11T06:42:30,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac9d5ce872254f7a879ae05464465314 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/ac9d5ce872254f7a879ae05464465314 2024-11-11T06:42:30,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/2362badf35a541cc91a0522f8300ab08 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/2362badf35a541cc91a0522f8300ab08 2024-11-11T06:42:30,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6412d197cf4041ecb505fda10ae57b00 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6412d197cf4041ecb505fda10ae57b00 2024-11-11T06:42:30,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7131f7a1b1db4e9bb2dc640499094913 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/7131f7a1b1db4e9bb2dc640499094913 2024-11-11T06:42:30,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/372a998c4b7a434289e8a903256eabf3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/372a998c4b7a434289e8a903256eabf3 2024-11-11T06:42:30,888 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=1 2024-11-11T06:42:30,888 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. 2024-11-11T06:42:30,888 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for c363d62112ab347adf2457471c10bf81: Waiting for close lock at 1731307349459Running coprocessor pre-close hooks at 1731307349459Disabling compacts and flushes for region at 1731307349459Disabling writes for close at 1731307349460 (+1 ms)Obtaining lock to block concurrent updates at 1731307349460Preparing flush snapshotting stores in c363d62112ab347adf2457471c10bf81 at 1731307349460Finished memstore snapshotting TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731307349461 (+1 ms)Flushing stores of TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81. at 1731307349462 (+1 ms)Flushing c363d62112ab347adf2457471c10bf81/A: creating writer at 1731307349462Flushing c363d62112ab347adf2457471c10bf81/A: appending metadata at 1731307349468 (+6 ms)Flushing c363d62112ab347adf2457471c10bf81/A: closing flushed file at 1731307349468Flushing c363d62112ab347adf2457471c10bf81/B: creating writer at 1731307349928 (+460 ms)Flushing c363d62112ab347adf2457471c10bf81/B: appending metadata at 1731307349933 (+5 ms)Flushing c363d62112ab347adf2457471c10bf81/B: closing flushed file at 1731307349933Flushing c363d62112ab347adf2457471c10bf81/C: creating writer at 1731307350342 (+409 ms)Flushing c363d62112ab347adf2457471c10bf81/C: appending metadata at 1731307350345 (+3 ms)Flushing c363d62112ab347adf2457471c10bf81/C: closing flushed file at 1731307350345Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eb50c03: reopening flushed file at 1731307350761 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f3987dd: reopening flushed file at 1731307350766 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34b02518: reopening flushed file at 1731307350771 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c363d62112ab347adf2457471c10bf81 in 1316ms, sequenceid=348, compaction requested=true at 1731307350776 (+5 ms)Writing region close event to WAL at 1731307350884 (+108 ms)Running coprocessor post-close hooks at 1731307350888 (+4 ms)Closed at 1731307350888 2024-11-11T06:42:30,891 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:30,891 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=c363d62112ab347adf2457471c10bf81, regionState=CLOSED 2024-11-11T06:42:30,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:42:30,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T06:42:30,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure c363d62112ab347adf2457471c10bf81, server=ce2cf0244921,35807,1731307183669 in 1.5890 sec 2024-11-11T06:42:30,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-11T06:42:30,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c363d62112ab347adf2457471c10bf81, UNASSIGN in 1.5930 sec 2024-11-11T06:42:30,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T06:42:30,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5980 sec 2024-11-11T06:42:30,902 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307350902"}]},"ts":"1731307350902"} 2024-11-11T06:42:30,903 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T06:42:30,904 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T06:42:30,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6110 sec 2024-11-11T06:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T06:42:31,440 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:31,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T06:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:31,444 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T06:42:31,445 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:31,448 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:31,450 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/recovered.edits] 2024-11-11T06:42:31,453 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/64c3872df6354240845ee9f99d2b07ad to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/64c3872df6354240845ee9f99d2b07ad 2024-11-11T06:42:31,454 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/9e77952b9f0e4fd28637f43b722535b3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/9e77952b9f0e4fd28637f43b722535b3 2024-11-11T06:42:31,455 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/f1151815026948de83218ee398a46e3d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/A/f1151815026948de83218ee398a46e3d 2024-11-11T06:42:31,458 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/127abb94117a4c1bbe888f08d29f3bb5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/127abb94117a4c1bbe888f08d29f3bb5 2024-11-11T06:42:31,460 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/33a7587acad34beb8149549053b9ba31 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/33a7587acad34beb8149549053b9ba31 2024-11-11T06:42:31,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/fa6f9367504a4d24ac6c1c27b09483e6 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/B/fa6f9367504a4d24ac6c1c27b09483e6 2024-11-11T06:42:31,465 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0bd43b5b014e4d47bf306bb6ad3e8806 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/0bd43b5b014e4d47bf306bb6ad3e8806 2024-11-11T06:42:31,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6571d960445a4de685ad55056aef3bd9 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/6571d960445a4de685ad55056aef3bd9 2024-11-11T06:42:31,468 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c7e14a8436054d908ba9598c5f0ae886 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/C/c7e14a8436054d908ba9598c5f0ae886 2024-11-11T06:42:31,472 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/recovered.edits/351.seqid to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81/recovered.edits/351.seqid 2024-11-11T06:42:31,473 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/c363d62112ab347adf2457471c10bf81 2024-11-11T06:42:31,473 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T06:42:31,476 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:31,480 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T06:42:31,482 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T06:42:31,483 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:31,484 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T06:42:31,484 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731307351484"}]},"ts":"9223372036854775807"} 2024-11-11T06:42:31,486 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T06:42:31,486 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c363d62112ab347adf2457471c10bf81, NAME => 'TestAcidGuarantees,,1731307323957.c363d62112ab347adf2457471c10bf81.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T06:42:31,486 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T06:42:31,486 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731307351486"}]},"ts":"9223372036854775807"} 2024-11-11T06:42:31,488 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T06:42:31,489 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:31,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-11T06:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T06:42:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T06:42:31,551 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:31,567 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241 (was 238) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=353 (was 382), ProcessCount=11 (was 11), AvailableMemoryMB=10851 (was 10866) 2024-11-11T06:42:31,576 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=353, ProcessCount=11, AvailableMemoryMB=10851 2024-11-11T06:42:31,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:42:31,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:31,579 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T06:42:31,580 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:31,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-11T06:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T06:42:31,580 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T06:42:31,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742322_1498 (size=959) 2024-11-11T06:42:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T06:42:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T06:42:31,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416 2024-11-11T06:42:32,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742323_1499 (size=53) 2024-11-11T06:42:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T06:42:32,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:42:32,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8d454a2f13681bc747ece68bc181872e, disabling compactions & flushes 2024-11-11T06:42:32,406 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:32,407 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:32,407 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. after waiting 0 ms 2024-11-11T06:42:32,407 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:32,407 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:32,407 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8d454a2f13681bc747ece68bc181872e: Waiting for close lock at 1731307352406Disabling compacts and flushes for region at 1731307352406Disabling writes for close at 1731307352407 (+1 ms)Writing region close event to WAL at 1731307352407Closed at 1731307352407 2024-11-11T06:42:32,410 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T06:42:32,411 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731307352410"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731307352410"}]},"ts":"1731307352410"} 2024-11-11T06:42:32,413 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T06:42:32,414 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T06:42:32,414 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307352414"}]},"ts":"1731307352414"} 2024-11-11T06:42:32,417 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T06:42:32,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, ASSIGN}] 2024-11-11T06:42:32,419 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, ASSIGN 2024-11-11T06:42:32,420 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, ASSIGN; state=OFFLINE, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=false 2024-11-11T06:42:32,572 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=8d454a2f13681bc747ece68bc181872e, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:32,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, ASSIGN because future has completed 2024-11-11T06:42:32,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:42:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T06:42:32,737 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:32,737 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:42:32,738 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,738 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:42:32,738 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,738 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,740 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,742 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d454a2f13681bc747ece68bc181872e columnFamilyName A 2024-11-11T06:42:32,742 DEBUG [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:32,743 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(327): Store=8d454a2f13681bc747ece68bc181872e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:32,743 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,745 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d454a2f13681bc747ece68bc181872e columnFamilyName B 2024-11-11T06:42:32,745 DEBUG [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:32,746 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(327): Store=8d454a2f13681bc747ece68bc181872e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:32,746 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,748 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d454a2f13681bc747ece68bc181872e columnFamilyName C 2024-11-11T06:42:32,748 DEBUG [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:32,748 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(327): Store=8d454a2f13681bc747ece68bc181872e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:32,749 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,750 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,750 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,752 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,752 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,753 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:42:32,755 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,759 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T06:42:32,760 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 8d454a2f13681bc747ece68bc181872e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65640232, jitterRate=-0.02188432216644287}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:42:32,760 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:32,761 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 8d454a2f13681bc747ece68bc181872e: Running coprocessor pre-open hook at 1731307352739Writing region info on filesystem at 1731307352739Initializing all the Stores at 1731307352740 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307352740Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307352740Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307352740Cleaning up temporary data from old regions at 1731307352752 (+12 ms)Running coprocessor post-open hooks at 1731307352760 (+8 ms)Region opened successfully at 1731307352761 (+1 ms) 2024-11-11T06:42:32,761 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., pid=140, masterSystemTime=1731307352731 2024-11-11T06:42:32,763 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:32,763 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:32,763 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=8d454a2f13681bc747ece68bc181872e, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:32,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:42:32,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T06:42:32,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 in 188 msec 2024-11-11T06:42:32,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-11T06:42:32,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, ASSIGN in 349 msec 2024-11-11T06:42:32,768 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T06:42:32,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307352768"}]},"ts":"1731307352768"} 2024-11-11T06:42:32,769 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T06:42:32,769 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T06:42:32,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-11T06:42:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T06:42:33,729 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:33,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:33,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:33,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:33,731 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc93e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:33,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a4ee97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:33,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:33,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:33,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:33,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:33,736 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T06:42:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,40845,1731307182847 2024-11-11T06:42:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@142d2978 2024-11-11T06:42:33,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T06:42:33,738 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T06:42:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T06:42:33,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T06:42:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:33,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742324_1500 (size=995) 2024-11-11T06:42:34,156 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T06:42:34,156 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T06:42:34,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:42:34,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, REOPEN/MOVE}] 2024-11-11T06:42:34,163 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, REOPEN/MOVE 2024-11-11T06:42:34,164 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8d454a2f13681bc747ece68bc181872e, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, REOPEN/MOVE because future has completed 2024-11-11T06:42:34,165 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:42:34,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:42:34,319 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,319 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:42:34,319 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 8d454a2f13681bc747ece68bc181872e, disabling compactions & flushes 2024-11-11T06:42:34,319 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,319 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,319 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. after waiting 0 ms 2024-11-11T06:42:34,319 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,327 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T06:42:34,328 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,328 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 8d454a2f13681bc747ece68bc181872e: Waiting for close lock at 1731307354319Running coprocessor pre-close hooks at 1731307354319Disabling compacts and flushes for region at 1731307354319Disabling writes for close at 1731307354319Writing region close event to WAL at 1731307354320 (+1 ms)Running coprocessor post-close hooks at 1731307354328 (+8 ms)Closed at 1731307354328 2024-11-11T06:42:34,328 WARN [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 8d454a2f13681bc747ece68bc181872e to self. 2024-11-11T06:42:34,330 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,331 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8d454a2f13681bc747ece68bc181872e, regionState=CLOSED 2024-11-11T06:42:34,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:42:34,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-11T06:42:34,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 in 169 msec 2024-11-11T06:42:34,337 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, REOPEN/MOVE; state=CLOSED, location=ce2cf0244921,35807,1731307183669; forceNewPlan=false, retain=true 2024-11-11T06:42:34,488 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8d454a2f13681bc747ece68bc181872e, regionState=OPENING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, REOPEN/MOVE because future has completed 2024-11-11T06:42:34,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:42:34,654 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,655 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T06:42:34,655 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,656 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T06:42:34,656 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,656 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,659 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,660 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d454a2f13681bc747ece68bc181872e columnFamilyName A 2024-11-11T06:42:34,662 DEBUG [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:34,663 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(327): Store=8d454a2f13681bc747ece68bc181872e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:34,663 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,664 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d454a2f13681bc747ece68bc181872e columnFamilyName B 2024-11-11T06:42:34,664 DEBUG [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:34,665 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(327): Store=8d454a2f13681bc747ece68bc181872e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:34,665 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,666 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d454a2f13681bc747ece68bc181872e columnFamilyName C 2024-11-11T06:42:34,666 DEBUG [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:34,667 INFO [StoreOpener-8d454a2f13681bc747ece68bc181872e-1 {}] regionserver.HStore(327): Store=8d454a2f13681bc747ece68bc181872e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T06:42:34,667 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,668 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,670 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,671 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,671 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,672 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T06:42:34,674 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,675 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 8d454a2f13681bc747ece68bc181872e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75247507, jitterRate=0.12127523124217987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T06:42:34,675 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,675 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 8d454a2f13681bc747ece68bc181872e: Running coprocessor pre-open hook at 1731307354656Writing region info on filesystem at 1731307354656Initializing all the Stores at 1731307354658 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307354658Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307354659 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731307354659Cleaning up temporary data from old regions at 1731307354671 (+12 ms)Running coprocessor post-open hooks at 1731307354675 (+4 ms)Region opened successfully at 1731307354675 2024-11-11T06:42:34,676 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., pid=145, masterSystemTime=1731307354647 2024-11-11T06:42:34,678 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,678 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,679 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8d454a2f13681bc747ece68bc181872e, regionState=OPEN, openSeqNum=5, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:42:34,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-11T06:42:34,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 in 189 msec 2024-11-11T06:42:34,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-11T06:42:34,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, REOPEN/MOVE in 523 msec 2024-11-11T06:42:34,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T06:42:34,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 529 msec 2024-11-11T06:42:34,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-11-11T06:42:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T06:42:34,694 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:34,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227f09c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28914654, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63b7e019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b1b09d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c52cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e0b31a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,704 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bc015f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,706 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30566f89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,707 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79939d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,708 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb93bc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,711 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4e2c30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f193c01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,714 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b702f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,716 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d1e764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655b109d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed7c77a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,722 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8fb318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,723 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3537405c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,40845,-1 for getting cluster id 2024-11-11T06:42:34,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T06:42:34,725 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16654907-934f-4fd9-82c1-8c338d43dded' 2024-11-11T06:42:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T06:42:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16654907-934f-4fd9-82c1-8c338d43dded" 2024-11-11T06:42:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70991f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,40845,-1] 2024-11-11T06:42:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T06:42:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:34,726 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T06:42:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197414db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T06:42:34,727 DEBUG [Thread-2269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,727 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,727 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,727 DEBUG [Thread-2258 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-11T06:42:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,728 DEBUG [Thread-2265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,729 DEBUG [Thread-2260 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T06:42:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,729 DEBUG [Thread-2254 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,730 DEBUG [Thread-2271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,730 DEBUG [Thread-2256 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,730 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,731 DEBUG [Thread-2262 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T06:42:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,740 DEBUG [Thread-2260 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,35807,1731307183669, seqNum=-1] 2024-11-11T06:42:34,741 DEBUG [Thread-2260 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T06:42:34,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T06:42:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5] 2024-11-11T06:42:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:34,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:34,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:42:34,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112752eaed3fb14c7cb4251726546d90a1_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307354740/Put/seqid=0 2024-11-11T06:42:34,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742325_1501 (size=12154) 2024-11-11T06:42:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307414793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307414793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307414794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307414794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307414795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T06:42:34,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:34,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:34,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:34,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:34,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:34,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307414908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307414908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307414909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307414909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307414909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:35,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:35,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T06:42:35,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307415119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307415119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307415119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307415119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307415119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,168 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:35,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112752eaed3fb14c7cb4251726546d90a1_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112752eaed3fb14c7cb4251726546d90a1_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:35,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/e171a62c62f3409289f50de13232dbb1, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:35,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/e171a62c62f3409289f50de13232dbb1 is 175, key is test_row_0/A:col10/1731307354740/Put/seqid=0 2024-11-11T06:42:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742326_1502 (size=30955) 2024-11-11T06:42:35,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:35,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:35,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:35,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:35,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T06:42:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307415429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307415429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307415429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307415429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307415430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,575 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/e171a62c62f3409289f50de13232dbb1 2024-11-11T06:42:35,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/c0a73f3cfdd6448baac10da226bf23bd is 50, key is test_row_0/B:col10/1731307354740/Put/seqid=0 2024-11-11T06:42:35,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742327_1503 (size=12001) 2024-11-11T06:42:35,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:35,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:35,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T06:42:35,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307415939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307415939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307415940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307415940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307415940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:35,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:35,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:35,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/c0a73f3cfdd6448baac10da226bf23bd 2024-11-11T06:42:36,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/df09468e8f5d4dfc9e55690113f995f9 is 50, key is test_row_0/C:col10/1731307354740/Put/seqid=0 2024-11-11T06:42:36,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742328_1504 (size=12001) 2024-11-11T06:42:36,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:36,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:36,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:36,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:36,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:36,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:36,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:36,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:36,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,406 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T06:42:36,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:36,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:36,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/df09468e8f5d4dfc9e55690113f995f9 2024-11-11T06:42:36,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/e171a62c62f3409289f50de13232dbb1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/e171a62c62f3409289f50de13232dbb1 2024-11-11T06:42:36,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/e171a62c62f3409289f50de13232dbb1, entries=150, sequenceid=16, filesize=30.2 K 2024-11-11T06:42:36,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/c0a73f3cfdd6448baac10da226bf23bd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/c0a73f3cfdd6448baac10da226bf23bd 2024-11-11T06:42:36,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/c0a73f3cfdd6448baac10da226bf23bd, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T06:42:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/df09468e8f5d4dfc9e55690113f995f9 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/df09468e8f5d4dfc9e55690113f995f9 2024-11-11T06:42:36,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/df09468e8f5d4dfc9e55690113f995f9, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T06:42:36,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8d454a2f13681bc747ece68bc181872e in 1687ms, sequenceid=16, compaction requested=false 2024-11-11T06:42:36,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:36,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T06:42:36,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:36,560 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T06:42:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111171fe9ec1e13447b38a555f80e6c43659_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307354794/Put/seqid=0 2024-11-11T06:42:36,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742329_1505 (size=12154) 2024-11-11T06:42:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T06:42:36,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:36,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307416960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:36,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307416961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:36,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307416963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307416964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307416964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:36,972 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111171fe9ec1e13447b38a555f80e6c43659_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111171fe9ec1e13447b38a555f80e6c43659_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/df04dffdf7814f1f87b8bbe30fd4710b, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/df04dffdf7814f1f87b8bbe30fd4710b is 175, key is test_row_0/A:col10/1731307354794/Put/seqid=0 2024-11-11T06:42:36,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742330_1506 (size=30955) 2024-11-11T06:42:37,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307417069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307417069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307417069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307417069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307417069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307417279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307417279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307417279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307417279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307417279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,375 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/df04dffdf7814f1f87b8bbe30fd4710b 2024-11-11T06:42:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/42b438eccfa849779273242c869349cd is 50, key is test_row_0/B:col10/1731307354794/Put/seqid=0 2024-11-11T06:42:37,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742331_1507 (size=12001) 2024-11-11T06:42:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307417588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307417588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307417589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307417589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307417589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:37,785 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/42b438eccfa849779273242c869349cd 2024-11-11T06:42:37,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/39516f2df5094487a21cc482f4dda9d9 is 50, key is test_row_0/C:col10/1731307354794/Put/seqid=0 2024-11-11T06:42:37,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742332_1508 (size=12001) 2024-11-11T06:42:38,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307418099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:38,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307418099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:38,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307418100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:38,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307418100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:38,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307418100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:38,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/39516f2df5094487a21cc482f4dda9d9 2024-11-11T06:42:38,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/df04dffdf7814f1f87b8bbe30fd4710b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/df04dffdf7814f1f87b8bbe30fd4710b 2024-11-11T06:42:38,199 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/df04dffdf7814f1f87b8bbe30fd4710b, entries=150, sequenceid=40, filesize=30.2 K 2024-11-11T06:42:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/42b438eccfa849779273242c869349cd as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/42b438eccfa849779273242c869349cd 2024-11-11T06:42:38,202 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/42b438eccfa849779273242c869349cd, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T06:42:38,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/39516f2df5094487a21cc482f4dda9d9 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/39516f2df5094487a21cc482f4dda9d9 2024-11-11T06:42:38,206 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/39516f2df5094487a21cc482f4dda9d9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T06:42:38,207 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8d454a2f13681bc747ece68bc181872e in 1646ms, sequenceid=40, compaction requested=false 2024-11-11T06:42:38,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:38,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:38,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-11T06:42:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-11T06:42:38,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T06:42:38,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4780 sec 2024-11-11T06:42:38,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.4830 sec 2024-11-11T06:42:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T06:42:38,900 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:38,900 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-11T06:42:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T06:42:38,901 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:38,902 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:38,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T06:42:39,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T06:42:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:39,054 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T06:42:39,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111855cbeea12a4f0bbb8acc1105fc325a_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307356958/Put/seqid=0 2024-11-11T06:42:39,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742333_1509 (size=12154) 2024-11-11T06:42:39,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:39,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307419129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307419130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307419132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307419134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307419134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T06:42:39,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307419238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307419238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307419238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307419249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307419249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307419449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307419449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307419449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307419459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307419459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:39,466 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111855cbeea12a4f0bbb8acc1105fc325a_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111855cbeea12a4f0bbb8acc1105fc325a_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:39,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/5eb663be0488476ab360a70446f6ce6b, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:39,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/5eb663be0488476ab360a70446f6ce6b is 175, key is test_row_0/A:col10/1731307356958/Put/seqid=0 2024-11-11T06:42:39,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742334_1510 (size=30955) 2024-11-11T06:42:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T06:42:39,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307419759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307419759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307419759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307419769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307419769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:39,870 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/5eb663be0488476ab360a70446f6ce6b 2024-11-11T06:42:39,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/0cfcc86f725f43e2a4ff9cb919633a7d is 50, key is test_row_0/B:col10/1731307356958/Put/seqid=0 2024-11-11T06:42:39,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742335_1511 (size=12001) 2024-11-11T06:42:39,878 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/0cfcc86f725f43e2a4ff9cb919633a7d 2024-11-11T06:42:39,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/f6393a7e350f40beac70363b49f346c3 is 50, key is test_row_0/C:col10/1731307356958/Put/seqid=0 2024-11-11T06:42:39,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742336_1512 (size=12001) 2024-11-11T06:42:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T06:42:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307420269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307420269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:40,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307420269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:40,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307420279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:40,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307420279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:40,335 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/f6393a7e350f40beac70363b49f346c3 2024-11-11T06:42:40,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/5eb663be0488476ab360a70446f6ce6b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5eb663be0488476ab360a70446f6ce6b 2024-11-11T06:42:40,341 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5eb663be0488476ab360a70446f6ce6b, entries=150, sequenceid=52, filesize=30.2 K 2024-11-11T06:42:40,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/0cfcc86f725f43e2a4ff9cb919633a7d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0cfcc86f725f43e2a4ff9cb919633a7d 2024-11-11T06:42:40,344 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0cfcc86f725f43e2a4ff9cb919633a7d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T06:42:40,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/f6393a7e350f40beac70363b49f346c3 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/f6393a7e350f40beac70363b49f346c3 2024-11-11T06:42:40,347 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/f6393a7e350f40beac70363b49f346c3, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T06:42:40,348 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8d454a2f13681bc747ece68bc181872e in 1294ms, sequenceid=52, compaction requested=true 2024-11-11T06:42:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T06:42:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-11T06:42:40,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T06:42:40,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-11T06:42:40,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.4510 sec 2024-11-11T06:42:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T06:42:41,049 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:41,050 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-11T06:42:41,051 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T06:42:41,051 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:41,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T06:42:41,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T06:42:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:41,203 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T06:42:41,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c4e490d953e84671ab0b8438d8890e07_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307359133/Put/seqid=0 2024-11-11T06:42:41,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742337_1513 (size=12154) 2024-11-11T06:42:41,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:41,219 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c4e490d953e84671ab0b8438d8890e07_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c4e490d953e84671ab0b8438d8890e07_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/1e443970822a46258f7462da9832ff52, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:41,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/1e443970822a46258f7462da9832ff52 is 175, key is test_row_0/A:col10/1731307359133/Put/seqid=0 2024-11-11T06:42:41,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742338_1514 (size=30955) 2024-11-11T06:42:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:41,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:41,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307421289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,293 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_0' in 8d454a2f13681bc747ece68bc181872e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307421292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307421292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307421292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307421299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,303 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_0' in 8d454a2f13681bc747ece68bc181872e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T06:42:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307421399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307421411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307421410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307421609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307421619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307421619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,623 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/1e443970822a46258f7462da9832ff52 2024-11-11T06:42:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/9ad23339090443f8b07a211dbdee95d8 is 50, key is test_row_0/B:col10/1731307359133/Put/seqid=0 2024-11-11T06:42:41,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742339_1515 (size=12001) 2024-11-11T06:42:41,645 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/9ad23339090443f8b07a211dbdee95d8 2024-11-11T06:42:41,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/8e7c057b497d4540b5ac52982d8c5170 is 50, key is test_row_0/C:col10/1731307359133/Put/seqid=0 2024-11-11T06:42:41,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742340_1516 (size=12001) 2024-11-11T06:42:41,665 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/8e7c057b497d4540b5ac52982d8c5170 2024-11-11T06:42:41,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/1e443970822a46258f7462da9832ff52 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/1e443970822a46258f7462da9832ff52 2024-11-11T06:42:41,674 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/1e443970822a46258f7462da9832ff52, entries=150, sequenceid=77, filesize=30.2 K 2024-11-11T06:42:41,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/9ad23339090443f8b07a211dbdee95d8 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/9ad23339090443f8b07a211dbdee95d8 2024-11-11T06:42:41,677 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/9ad23339090443f8b07a211dbdee95d8, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T06:42:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/8e7c057b497d4540b5ac52982d8c5170 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8e7c057b497d4540b5ac52982d8c5170 2024-11-11T06:42:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T06:42:41,681 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8e7c057b497d4540b5ac52982d8c5170, entries=150, sequenceid=77, filesize=11.7 K 2024-11-11T06:42:41,682 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 8d454a2f13681bc747ece68bc181872e in 478ms, sequenceid=77, compaction requested=true 2024-11-11T06:42:41,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:41,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:41,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-11T06:42:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-11T06:42:41,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T06:42:41,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 632 msec 2024-11-11T06:42:41,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 634 msec 2024-11-11T06:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:41,921 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:41,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T06:42:41,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111149c8b8b2fe8b40399560b99dd1c3e2fb_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307361920/Put/seqid=0 2024-11-11T06:42:41,928 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T06:42:41,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742341_1517 (size=14594) 2024-11-11T06:42:41,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307421960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307421965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307421966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307422069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307422079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307422079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T06:42:42,189 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:42,190 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T06:42:42,191 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T06:42:42,191 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:42,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:42,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307422279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307422289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307422289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T06:42:42,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:42,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111149c8b8b2fe8b40399560b99dd1c3e2fb_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149c8b8b2fe8b40399560b99dd1c3e2fb_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:42,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fd4b689301f54bcd81bd85d207bc63f7, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:42,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fd4b689301f54bcd81bd85d207bc63f7 is 175, key is test_row_0/A:col10/1731307361920/Put/seqid=0 2024-11-11T06:42:42,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742342_1518 (size=39549) 2024-11-11T06:42:42,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:42,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:42,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:42,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:42,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T06:42:42,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307422589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307422598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307422599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:42,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:42,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:42,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,738 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fd4b689301f54bcd81bd85d207bc63f7 2024-11-11T06:42:42,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/3694222e3eae45a4a2902b5eafefefb0 is 50, key is test_row_0/B:col10/1731307361920/Put/seqid=0 2024-11-11T06:42:42,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742343_1519 (size=12001) 2024-11-11T06:42:42,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T06:42:42,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:42,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:42,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307423099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:43,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307423108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:43,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307423109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:43,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/3694222e3eae45a4a2902b5eafefefb0 2024-11-11T06:42:43,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/8c1379ab816540d7a2aadfedc0c07b5b is 50, key is test_row_0/C:col10/1731307361920/Put/seqid=0 2024-11-11T06:42:43,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742344_1520 (size=12001) 2024-11-11T06:42:43,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307423319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:43,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307423319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:43,322 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_0' in 8d454a2f13681bc747ece68bc181872e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,322 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_0' in 8d454a2f13681bc747ece68bc181872e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:43,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T06:42:43,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:43,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:43,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/8c1379ab816540d7a2aadfedc0c07b5b 2024-11-11T06:42:43,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fd4b689301f54bcd81bd85d207bc63f7 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fd4b689301f54bcd81bd85d207bc63f7 2024-11-11T06:42:43,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fd4b689301f54bcd81bd85d207bc63f7, entries=200, sequenceid=90, filesize=38.6 K 2024-11-11T06:42:43,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/3694222e3eae45a4a2902b5eafefefb0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/3694222e3eae45a4a2902b5eafefefb0 2024-11-11T06:42:43,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:43,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/3694222e3eae45a4a2902b5eafefefb0, entries=150, sequenceid=90, filesize=11.7 K 2024-11-11T06:42:43,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/8c1379ab816540d7a2aadfedc0c07b5b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8c1379ab816540d7a2aadfedc0c07b5b 2024-11-11T06:42:43,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8c1379ab816540d7a2aadfedc0c07b5b, entries=150, sequenceid=90, filesize=11.7 K 2024-11-11T06:42:43,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8d454a2f13681bc747ece68bc181872e in 1648ms, sequenceid=90, compaction requested=true 2024-11-11T06:42:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:43,569 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T06:42:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:43,569 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T06:42:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:43,570 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163369 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T06:42:43,570 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T06:42:43,570 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/B is initiating minor compaction (all files) 2024-11-11T06:42:43,570 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/A is initiating minor compaction (all files) 2024-11-11T06:42:43,570 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/B in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,570 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/A in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,570 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/c0a73f3cfdd6448baac10da226bf23bd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/42b438eccfa849779273242c869349cd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0cfcc86f725f43e2a4ff9cb919633a7d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/9ad23339090443f8b07a211dbdee95d8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/3694222e3eae45a4a2902b5eafefefb0] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=58.6 K 2024-11-11T06:42:43,570 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/e171a62c62f3409289f50de13232dbb1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/df04dffdf7814f1f87b8bbe30fd4710b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5eb663be0488476ab360a70446f6ce6b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/1e443970822a46258f7462da9832ff52, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fd4b689301f54bcd81bd85d207bc63f7] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=159.5 K 2024-11-11T06:42:43,570 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,570 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/e171a62c62f3409289f50de13232dbb1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/df04dffdf7814f1f87b8bbe30fd4710b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5eb663be0488476ab360a70446f6ce6b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/1e443970822a46258f7462da9832ff52, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fd4b689301f54bcd81bd85d207bc63f7] 2024-11-11T06:42:43,571 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting c0a73f3cfdd6448baac10da226bf23bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731307354740 2024-11-11T06:42:43,571 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting e171a62c62f3409289f50de13232dbb1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731307354740 2024-11-11T06:42:43,571 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b438eccfa849779273242c869349cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731307354792 2024-11-11T06:42:43,571 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting df04dffdf7814f1f87b8bbe30fd4710b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731307354792 2024-11-11T06:42:43,571 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cfcc86f725f43e2a4ff9cb919633a7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307356958 2024-11-11T06:42:43,571 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad23339090443f8b07a211dbdee95d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731307359129 2024-11-11T06:42:43,571 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eb663be0488476ab360a70446f6ce6b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307356958 2024-11-11T06:42:43,572 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 3694222e3eae45a4a2902b5eafefefb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731307361285 2024-11-11T06:42:43,572 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e443970822a46258f7462da9832ff52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731307359129 2024-11-11T06:42:43,572 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd4b689301f54bcd81bd85d207bc63f7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731307361285 2024-11-11T06:42:43,577 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:43,578 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115f65b1d7744f409581f3162e3dd78c93_8d454a2f13681bc747ece68bc181872e store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:43,578 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:43,579 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/eb099bcdab61403c91ca736ecc460aa1 is 50, key is test_row_0/B:col10/1731307361920/Put/seqid=0 2024-11-11T06:42:43,580 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115f65b1d7744f409581f3162e3dd78c93_8d454a2f13681bc747ece68bc181872e, store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:43,581 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115f65b1d7744f409581f3162e3dd78c93_8d454a2f13681bc747ece68bc181872e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:43,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742345_1521 (size=12173) 2024-11-11T06:42:43,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742346_1522 (size=4469) 2024-11-11T06:42:43,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T06:42:43,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,716 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T06:42:43,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117ed0a11d901a4b638b301e5f10292bf4_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307361964/Put/seqid=0 2024-11-11T06:42:43,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742347_1523 (size=12154) 2024-11-11T06:42:43,990 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/eb099bcdab61403c91ca736ecc460aa1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/eb099bcdab61403c91ca736ecc460aa1 2024-11-11T06:42:43,992 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#A#compaction#439 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:43,992 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/b2d2898b5f024620a544b3a409ffbaca is 175, key is test_row_0/A:col10/1731307361920/Put/seqid=0 2024-11-11T06:42:43,993 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/B of 8d454a2f13681bc747ece68bc181872e into eb099bcdab61403c91ca736ecc460aa1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:43,993 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:43,993 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/B, priority=11, startTime=1731307363569; duration=0sec 2024-11-11T06:42:43,993 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:43,993 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:B 2024-11-11T06:42:43,993 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T06:42:43,996 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T06:42:43,996 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/C is initiating minor compaction (all files) 2024-11-11T06:42:43,996 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/C in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:43,996 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/df09468e8f5d4dfc9e55690113f995f9, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/39516f2df5094487a21cc482f4dda9d9, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/f6393a7e350f40beac70363b49f346c3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8e7c057b497d4540b5ac52982d8c5170, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8c1379ab816540d7a2aadfedc0c07b5b] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=58.6 K 2024-11-11T06:42:43,996 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting df09468e8f5d4dfc9e55690113f995f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731307354740 2024-11-11T06:42:43,996 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 39516f2df5094487a21cc482f4dda9d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731307354792 2024-11-11T06:42:43,997 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting f6393a7e350f40beac70363b49f346c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731307356958 2024-11-11T06:42:43,997 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7c057b497d4540b5ac52982d8c5170, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731307359129 2024-11-11T06:42:43,997 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c1379ab816540d7a2aadfedc0c07b5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731307361285 2024-11-11T06:42:44,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742348_1524 (size=31127) 2024-11-11T06:42:44,010 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#C#compaction#441 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:44,010 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/97cf153313d14df0838118a43b98d608 is 50, key is test_row_0/C:col10/1731307361920/Put/seqid=0 2024-11-11T06:42:44,012 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/b2d2898b5f024620a544b3a409ffbaca as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/b2d2898b5f024620a544b3a409ffbaca 2024-11-11T06:42:44,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742349_1525 (size=12173) 2024-11-11T06:42:44,016 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/A of 8d454a2f13681bc747ece68bc181872e into b2d2898b5f024620a544b3a409ffbaca(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:44,016 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:44,016 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/A, priority=11, startTime=1731307363569; duration=0sec 2024-11-11T06:42:44,016 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:44,016 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:A 2024-11-11T06:42:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:44,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:44,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:44,128 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117ed0a11d901a4b638b301e5f10292bf4_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ed0a11d901a4b638b301e5f10292bf4_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:44,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/9d8d22a2bf42471a9a0c7b0436e6d1fe, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:44,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/9d8d22a2bf42471a9a0c7b0436e6d1fe is 175, key is test_row_0/A:col10/1731307361964/Put/seqid=0 2024-11-11T06:42:44,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742350_1526 (size=30955) 2024-11-11T06:42:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307424130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307424131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307424132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307424239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307424239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307424239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T06:42:44,416 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/97cf153313d14df0838118a43b98d608 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/97cf153313d14df0838118a43b98d608 2024-11-11T06:42:44,420 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/C of 8d454a2f13681bc747ece68bc181872e into 97cf153313d14df0838118a43b98d608(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:44,420 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:44,420 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/C, priority=11, startTime=1731307363569; duration=0sec 2024-11-11T06:42:44,420 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:44,420 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:C 2024-11-11T06:42:44,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307424449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307424449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307424449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,532 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/9d8d22a2bf42471a9a0c7b0436e6d1fe 2024-11-11T06:42:44,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/ba9bce30660e4ca199b465f535503d8e is 50, key is test_row_0/B:col10/1731307361964/Put/seqid=0 2024-11-11T06:42:44,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742351_1527 (size=12001) 2024-11-11T06:42:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307424759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307424759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307424759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:44,942 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/ba9bce30660e4ca199b465f535503d8e 2024-11-11T06:42:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b2b6df3f14a542169ab0f2b8ae31909c is 50, key is test_row_0/C:col10/1731307361964/Put/seqid=0 2024-11-11T06:42:44,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742352_1528 (size=12001) 2024-11-11T06:42:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307425268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307425269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307425269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:45,394 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b2b6df3f14a542169ab0f2b8ae31909c 2024-11-11T06:42:45,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/9d8d22a2bf42471a9a0c7b0436e6d1fe as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/9d8d22a2bf42471a9a0c7b0436e6d1fe 2024-11-11T06:42:45,400 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/9d8d22a2bf42471a9a0c7b0436e6d1fe, entries=150, sequenceid=113, filesize=30.2 K 2024-11-11T06:42:45,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/ba9bce30660e4ca199b465f535503d8e as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/ba9bce30660e4ca199b465f535503d8e 2024-11-11T06:42:45,403 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/ba9bce30660e4ca199b465f535503d8e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-11T06:42:45,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b2b6df3f14a542169ab0f2b8ae31909c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b2b6df3f14a542169ab0f2b8ae31909c 2024-11-11T06:42:45,406 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b2b6df3f14a542169ab0f2b8ae31909c, entries=150, sequenceid=113, filesize=11.7 K 2024-11-11T06:42:45,407 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8d454a2f13681bc747ece68bc181872e in 1690ms, sequenceid=113, compaction requested=false 2024-11-11T06:42:45,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:45,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:45,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T06:42:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T06:42:45,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T06:42:45,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2160 sec 2024-11-11T06:42:45,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.2200 sec 2024-11-11T06:42:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:46,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:46,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T06:42:46,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aacf13bf982d46e382eeb53884568583_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307366280/Put/seqid=0 2024-11-11T06:42:46,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742353_1529 (size=14694) 2024-11-11T06:42:46,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307426340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307426340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307426341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T06:42:46,349 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:46,350 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T06:42:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T06:42:46,351 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:46,352 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:46,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:46,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307426449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307426449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307426449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T06:42:46,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:46,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:46,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307426659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307426659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307426659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T06:42:46,690 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:46,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aacf13bf982d46e382eeb53884568583_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aacf13bf982d46e382eeb53884568583_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:46,693 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/06a6599886a644779ebf760877521c14, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:46,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/06a6599886a644779ebf760877521c14 is 175, key is test_row_0/A:col10/1731307366280/Put/seqid=0 2024-11-11T06:42:46,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742354_1530 (size=39649) 2024-11-11T06:42:46,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:46,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:46,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:46,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:46,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:46,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:46,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307426969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307426969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307426969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:46,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T06:42:47,097 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/06a6599886a644779ebf760877521c14 2024-11-11T06:42:47,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/36239d72cd6d4f559d495a54d1664492 is 50, key is test_row_0/B:col10/1731307366280/Put/seqid=0 2024-11-11T06:42:47,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742355_1531 (size=12101) 2024-11-11T06:42:47,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:47,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:47,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:47,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:47,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40254 deadline: 1731307427329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:47,331 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_0' in 8d454a2f13681bc747ece68bc181872e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:47,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40248 deadline: 1731307427359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:47,361 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,35807,1731307183669 for 'test_row_0' in 8d454a2f13681bc747ece68bc181872e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T06:42:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:47,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307427479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307427479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307427479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T06:42:47,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/36239d72cd6d4f559d495a54d1664492 2024-11-11T06:42:47,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b03eb1cf072b4bd5b50c04a40ff847c5 is 50, key is test_row_0/C:col10/1731307366280/Put/seqid=0 2024-11-11T06:42:47,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742356_1532 (size=12101) 2024-11-11T06:42:47,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:47,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:47,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:47,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b03eb1cf072b4bd5b50c04a40ff847c5 2024-11-11T06:42:47,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/06a6599886a644779ebf760877521c14 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/06a6599886a644779ebf760877521c14 2024-11-11T06:42:47,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/06a6599886a644779ebf760877521c14, entries=200, sequenceid=131, filesize=38.7 K 2024-11-11T06:42:47,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/36239d72cd6d4f559d495a54d1664492 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/36239d72cd6d4f559d495a54d1664492 2024-11-11T06:42:47,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/36239d72cd6d4f559d495a54d1664492, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T06:42:47,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b03eb1cf072b4bd5b50c04a40ff847c5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b03eb1cf072b4bd5b50c04a40ff847c5 2024-11-11T06:42:47,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b03eb1cf072b4bd5b50c04a40ff847c5, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T06:42:47,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 8d454a2f13681bc747ece68bc181872e in 1644ms, sequenceid=131, compaction requested=true 2024-11-11T06:42:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:47,925 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:47,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:47,925 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:47,926 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36275 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:47,926 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101731 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:47,926 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/B is initiating minor compaction (all files) 2024-11-11T06:42:47,926 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/A is initiating minor compaction (all files) 2024-11-11T06:42:47,926 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/A in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,926 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/B in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,926 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/eb099bcdab61403c91ca736ecc460aa1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/ba9bce30660e4ca199b465f535503d8e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/36239d72cd6d4f559d495a54d1664492] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=35.4 K 2024-11-11T06:42:47,926 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/b2d2898b5f024620a544b3a409ffbaca, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/9d8d22a2bf42471a9a0c7b0436e6d1fe, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/06a6599886a644779ebf760877521c14] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=99.3 K 2024-11-11T06:42:47,926 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:47,926 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/b2d2898b5f024620a544b3a409ffbaca, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/9d8d22a2bf42471a9a0c7b0436e6d1fe, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/06a6599886a644779ebf760877521c14] 2024-11-11T06:42:47,926 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d2898b5f024620a544b3a409ffbaca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731307361285 2024-11-11T06:42:47,926 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting eb099bcdab61403c91ca736ecc460aa1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731307361285 2024-11-11T06:42:47,927 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9bce30660e4ca199b465f535503d8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731307361957 2024-11-11T06:42:47,927 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d8d22a2bf42471a9a0c7b0436e6d1fe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731307361957 2024-11-11T06:42:47,927 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 36239d72cd6d4f559d495a54d1664492, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307364121 2024-11-11T06:42:47,927 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a6599886a644779ebf760877521c14, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307364121 2024-11-11T06:42:47,931 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:47,932 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:47,933 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/e079913a84154dc589fa34eef893d134 is 50, key is test_row_0/B:col10/1731307366280/Put/seqid=0 2024-11-11T06:42:47,934 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111347b5bedaf7c4c82bb2a43de2bce3846_8d454a2f13681bc747ece68bc181872e store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:47,937 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111347b5bedaf7c4c82bb2a43de2bce3846_8d454a2f13681bc747ece68bc181872e, store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:47,937 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111347b5bedaf7c4c82bb2a43de2bce3846_8d454a2f13681bc747ece68bc181872e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:47,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742357_1533 (size=12375) 2024-11-11T06:42:47,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742358_1534 (size=4469) 2024-11-11T06:42:48,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T06:42:48,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:48,028 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T06:42:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117127da53b5f141ff905291b8c2bae8b7_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307366339/Put/seqid=0 2024-11-11T06:42:48,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742359_1535 (size=12304) 2024-11-11T06:42:48,341 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#A#compaction#448 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:48,341 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/2ce97949350f4100a50b82a325cc8cda is 175, key is test_row_0/A:col10/1731307366280/Put/seqid=0 2024-11-11T06:42:48,343 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/e079913a84154dc589fa34eef893d134 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/e079913a84154dc589fa34eef893d134 2024-11-11T06:42:48,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742360_1536 (size=31329) 2024-11-11T06:42:48,348 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/B of 8d454a2f13681bc747ece68bc181872e into e079913a84154dc589fa34eef893d134(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:48,348 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:48,348 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/B, priority=13, startTime=1731307367925; duration=0sec 2024-11-11T06:42:48,348 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:48,348 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:B 2024-11-11T06:42:48,348 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:48,348 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/2ce97949350f4100a50b82a325cc8cda as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/2ce97949350f4100a50b82a325cc8cda 2024-11-11T06:42:48,349 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36275 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:48,349 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/C is initiating minor compaction (all files) 2024-11-11T06:42:48,349 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/C in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:48,349 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/97cf153313d14df0838118a43b98d608, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b2b6df3f14a542169ab0f2b8ae31909c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b03eb1cf072b4bd5b50c04a40ff847c5] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=35.4 K 2024-11-11T06:42:48,349 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 97cf153313d14df0838118a43b98d608, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731307361285 2024-11-11T06:42:48,349 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b2b6df3f14a542169ab0f2b8ae31909c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731307361957 2024-11-11T06:42:48,350 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b03eb1cf072b4bd5b50c04a40ff847c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307364121 2024-11-11T06:42:48,352 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/A of 8d454a2f13681bc747ece68bc181872e into 2ce97949350f4100a50b82a325cc8cda(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:48,352 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:48,352 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/A, priority=13, startTime=1731307367925; duration=0sec 2024-11-11T06:42:48,352 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:48,352 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:A 2024-11-11T06:42:48,356 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#C#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:48,356 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/76dfe4bf70264fd08f9888d8878fb66d is 50, key is test_row_0/C:col10/1731307366280/Put/seqid=0 2024-11-11T06:42:48,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742361_1537 (size=12375) 2024-11-11T06:42:48,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:48,441 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117127da53b5f141ff905291b8c2bae8b7_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117127da53b5f141ff905291b8c2bae8b7_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/d47982b3d1334685881f06c1015cd747, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/d47982b3d1334685881f06c1015cd747 is 175, key is test_row_0/A:col10/1731307366339/Put/seqid=0 2024-11-11T06:42:48,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742362_1538 (size=31105) 2024-11-11T06:42:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:48,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T06:42:48,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307428501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307428504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307428505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307428608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307428609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307428619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,762 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/76dfe4bf70264fd08f9888d8878fb66d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/76dfe4bf70264fd08f9888d8878fb66d 2024-11-11T06:42:48,765 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/C of 8d454a2f13681bc747ece68bc181872e into 76dfe4bf70264fd08f9888d8878fb66d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:48,765 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:48,765 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/C, priority=13, startTime=1731307367925; duration=0sec 2024-11-11T06:42:48,766 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:48,766 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:C 2024-11-11T06:42:48,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307428819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307428819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307428828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:48,846 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/d47982b3d1334685881f06c1015cd747 2024-11-11T06:42:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/a0ae0761fbfc4cd5b905775e979d274f is 50, key is test_row_0/B:col10/1731307366339/Put/seqid=0 2024-11-11T06:42:48,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742363_1539 (size=12151) 2024-11-11T06:42:49,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307429129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307429129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307429139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,255 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/a0ae0761fbfc4cd5b905775e979d274f 2024-11-11T06:42:49,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/5fc59227991f4b34aa70995b8f7317d5 is 50, key is test_row_0/C:col10/1731307366339/Put/seqid=0 2024-11-11T06:42:49,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742364_1540 (size=12151) 2024-11-11T06:42:49,267 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/5fc59227991f4b34aa70995b8f7317d5 2024-11-11T06:42:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/d47982b3d1334685881f06c1015cd747 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/d47982b3d1334685881f06c1015cd747 2024-11-11T06:42:49,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/d47982b3d1334685881f06c1015cd747, entries=150, sequenceid=153, filesize=30.4 K 2024-11-11T06:42:49,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/a0ae0761fbfc4cd5b905775e979d274f as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/a0ae0761fbfc4cd5b905775e979d274f 2024-11-11T06:42:49,276 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/a0ae0761fbfc4cd5b905775e979d274f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T06:42:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/5fc59227991f4b34aa70995b8f7317d5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/5fc59227991f4b34aa70995b8f7317d5 2024-11-11T06:42:49,278 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/5fc59227991f4b34aa70995b8f7317d5, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T06:42:49,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 8d454a2f13681bc747ece68bc181872e in 1251ms, sequenceid=153, compaction requested=false 2024-11-11T06:42:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:49,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T06:42:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T06:42:49,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T06:42:49,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9290 sec 2024-11-11T06:42:49,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.9330 sec 2024-11-11T06:42:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:49,641 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:49,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T06:42:49,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111955b6c307a184b5f9da2d3f305697e90_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307369641/Put/seqid=0 2024-11-11T06:42:49,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742365_1541 (size=14794) 2024-11-11T06:42:49,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307429670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307429675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307429676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307429779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307429789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307429789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:49,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307429988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307429999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:50,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307429999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,051 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:50,107 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111955b6c307a184b5f9da2d3f305697e90_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111955b6c307a184b5f9da2d3f305697e90_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:50,107 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fc65dcf89514492dbd093abbcab28287, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:50,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fc65dcf89514492dbd093abbcab28287 is 175, key is test_row_0/A:col10/1731307369641/Put/seqid=0 2024-11-11T06:42:50,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742366_1542 (size=39749) 2024-11-11T06:42:50,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307430299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307430309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:50,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307430309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fc65dcf89514492dbd093abbcab28287 2024-11-11T06:42:50,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/bdcecc07b1564d01b1f3e81678e2aab0 is 50, key is test_row_0/B:col10/1731307369641/Put/seqid=0 2024-11-11T06:42:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T06:42:50,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742367_1543 (size=12151) 2024-11-11T06:42:50,519 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:50,519 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-11T06:42:50,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:50,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:50,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T06:42:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T06:42:50,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T06:42:50,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:50,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:50,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:50,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:50,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307430809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307430819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:50,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307430819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:50,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T06:42:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:50,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T06:42:50,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/bdcecc07b1564d01b1f3e81678e2aab0 2024-11-11T06:42:50,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/4122a4f508584ec0a6868a3fc9ab18b6 is 50, key is test_row_0/C:col10/1731307369641/Put/seqid=0 2024-11-11T06:42:50,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742368_1544 (size=12151) 2024-11-11T06:42:50,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T06:42:50,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:50,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:50,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:50,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:50,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:51,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T06:42:51,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:51,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:51,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T06:42:51,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T06:42:51,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:51,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:51,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T06:42:51,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/4122a4f508584ec0a6868a3fc9ab18b6 2024-11-11T06:42:51,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fc65dcf89514492dbd093abbcab28287 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fc65dcf89514492dbd093abbcab28287 2024-11-11T06:42:51,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fc65dcf89514492dbd093abbcab28287, entries=200, sequenceid=172, filesize=38.8 K 2024-11-11T06:42:51,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/bdcecc07b1564d01b1f3e81678e2aab0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/bdcecc07b1564d01b1f3e81678e2aab0 2024-11-11T06:42:51,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/bdcecc07b1564d01b1f3e81678e2aab0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T06:42:51,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/4122a4f508584ec0a6868a3fc9ab18b6 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/4122a4f508584ec0a6868a3fc9ab18b6 2024-11-11T06:42:51,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/4122a4f508584ec0a6868a3fc9ab18b6, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T06:42:51,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8d454a2f13681bc747ece68bc181872e in 1701ms, sequenceid=172, compaction requested=true 2024-11-11T06:42:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:51,342 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:51,342 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/B is initiating minor compaction (all files) 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/A is initiating minor compaction (all files) 2024-11-11T06:42:51,343 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/A in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,343 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/B in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,343 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/e079913a84154dc589fa34eef893d134, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/a0ae0761fbfc4cd5b905775e979d274f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/bdcecc07b1564d01b1f3e81678e2aab0] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=35.8 K 2024-11-11T06:42:51,343 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/2ce97949350f4100a50b82a325cc8cda, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/d47982b3d1334685881f06c1015cd747, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fc65dcf89514492dbd093abbcab28287] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=99.8 K 2024-11-11T06:42:51,343 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/2ce97949350f4100a50b82a325cc8cda, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/d47982b3d1334685881f06c1015cd747, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fc65dcf89514492dbd093abbcab28287] 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting e079913a84154dc589fa34eef893d134, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307364121 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ce97949350f4100a50b82a325cc8cda, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307364121 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ae0761fbfc4cd5b905775e979d274f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731307366333 2024-11-11T06:42:51,343 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47982b3d1334685881f06c1015cd747, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731307366333 2024-11-11T06:42:51,344 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc65dcf89514492dbd093abbcab28287, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307368500 2024-11-11T06:42:51,344 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting bdcecc07b1564d01b1f3e81678e2aab0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307368504 2024-11-11T06:42:51,350 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:51,352 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:51,352 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/0af3d5a5fb60415ebd98a1b196eb2cad is 50, key is test_row_0/B:col10/1731307369641/Put/seqid=0 2024-11-11T06:42:51,358 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111110363466269046979c991909b0f0f1d2_8d454a2f13681bc747ece68bc181872e store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:51,359 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111110363466269046979c991909b0f0f1d2_8d454a2f13681bc747ece68bc181872e, store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:51,359 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110363466269046979c991909b0f0f1d2_8d454a2f13681bc747ece68bc181872e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742369_1545 (size=12527) 2024-11-11T06:42:51,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742370_1546 (size=4469) 2024-11-11T06:42:51,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T06:42:51,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,436 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T06:42:51,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111181d52800647b44498652ce791aad98ae_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307369675/Put/seqid=0 2024-11-11T06:42:51,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742371_1547 (size=12304) 2024-11-11T06:42:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T06:42:51,768 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/0af3d5a5fb60415ebd98a1b196eb2cad as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0af3d5a5fb60415ebd98a1b196eb2cad 2024-11-11T06:42:51,768 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#A#compaction#457 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:51,769 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/49b04b00eec24f6ea5d861767c6f24b6 is 175, key is test_row_0/A:col10/1731307369641/Put/seqid=0 2024-11-11T06:42:51,772 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/B of 8d454a2f13681bc747ece68bc181872e into 0af3d5a5fb60415ebd98a1b196eb2cad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:51,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742372_1548 (size=31481) 2024-11-11T06:42:51,772 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:51,772 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/B, priority=13, startTime=1731307371342; duration=0sec 2024-11-11T06:42:51,772 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:51,772 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:B 2024-11-11T06:42:51,772 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:51,773 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:51,773 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/C is initiating minor compaction (all files) 2024-11-11T06:42:51,773 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/C in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:51,773 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/76dfe4bf70264fd08f9888d8878fb66d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/5fc59227991f4b34aa70995b8f7317d5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/4122a4f508584ec0a6868a3fc9ab18b6] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=35.8 K 2024-11-11T06:42:51,773 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 76dfe4bf70264fd08f9888d8878fb66d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731307364121 2024-11-11T06:42:51,773 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc59227991f4b34aa70995b8f7317d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731307366333 2024-11-11T06:42:51,773 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 4122a4f508584ec0a6868a3fc9ab18b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307368504 2024-11-11T06:42:51,779 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#C#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:51,779 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/fb077e550b7441b78ee4e2aebd1d6e86 is 50, key is test_row_0/C:col10/1731307369641/Put/seqid=0 2024-11-11T06:42:51,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742373_1549 (size=12527) 2024-11-11T06:42:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:51,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:51,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307431843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:51,847 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111181d52800647b44498652ce791aad98ae_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111181d52800647b44498652ce791aad98ae_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:51,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307431844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:51,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/a10ac059ae5c46b884730b9e498ac67c, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/a10ac059ae5c46b884730b9e498ac67c is 175, key is test_row_0/A:col10/1731307369675/Put/seqid=0 2024-11-11T06:42:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307431846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:51,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742374_1550 (size=31105) 2024-11-11T06:42:51,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307431949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:51,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307431959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:51,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307431959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:52,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307432159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:52,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307432169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:52,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307432169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:52,175 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/49b04b00eec24f6ea5d861767c6f24b6 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/49b04b00eec24f6ea5d861767c6f24b6 2024-11-11T06:42:52,178 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/A of 8d454a2f13681bc747ece68bc181872e into 49b04b00eec24f6ea5d861767c6f24b6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:52,178 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:52,178 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/A, priority=13, startTime=1731307371342; duration=0sec 2024-11-11T06:42:52,178 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:52,178 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:A 2024-11-11T06:42:52,186 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/fb077e550b7441b78ee4e2aebd1d6e86 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/fb077e550b7441b78ee4e2aebd1d6e86 2024-11-11T06:42:52,189 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/C of 8d454a2f13681bc747ece68bc181872e into fb077e550b7441b78ee4e2aebd1d6e86(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:52,189 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:52,189 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/C, priority=13, startTime=1731307371342; duration=0sec 2024-11-11T06:42:52,189 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:52,189 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:C 2024-11-11T06:42:52,252 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/a10ac059ae5c46b884730b9e498ac67c 2024-11-11T06:42:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/277f7be4808c4b1082b74f0eb38906d9 is 50, key is test_row_0/B:col10/1731307369675/Put/seqid=0 2024-11-11T06:42:52,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742375_1551 (size=12151) 2024-11-11T06:42:52,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307432469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307432479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307432479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:52,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/277f7be4808c4b1082b74f0eb38906d9 2024-11-11T06:42:52,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b3fc4052eb0c416088a96e05da6ff93c is 50, key is test_row_0/C:col10/1731307369675/Put/seqid=0 2024-11-11T06:42:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T06:42:52,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742376_1552 (size=12151) 2024-11-11T06:42:52,669 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b3fc4052eb0c416088a96e05da6ff93c 2024-11-11T06:42:52,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/a10ac059ae5c46b884730b9e498ac67c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/a10ac059ae5c46b884730b9e498ac67c 2024-11-11T06:42:52,675 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/a10ac059ae5c46b884730b9e498ac67c, entries=150, sequenceid=192, filesize=30.4 K 2024-11-11T06:42:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/277f7be4808c4b1082b74f0eb38906d9 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/277f7be4808c4b1082b74f0eb38906d9 2024-11-11T06:42:52,678 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/277f7be4808c4b1082b74f0eb38906d9, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T06:42:52,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/b3fc4052eb0c416088a96e05da6ff93c as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b3fc4052eb0c416088a96e05da6ff93c 2024-11-11T06:42:52,680 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b3fc4052eb0c416088a96e05da6ff93c, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T06:42:52,681 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 8d454a2f13681bc747ece68bc181872e in 1245ms, sequenceid=192, compaction requested=false 2024-11-11T06:42:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-11T06:42:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-11T06:42:52,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T06:42:52,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-11-11T06:42:52,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.1650 sec 2024-11-11T06:42:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:52,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T06:42:52,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T06:42:52,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115bc7292b32b7442aadca724e68c55f3f_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307371840/Put/seqid=0 2024-11-11T06:42:52,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742377_1553 (size=14794) 2024-11-11T06:42:52,990 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:52,993 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115bc7292b32b7442aadca724e68c55f3f_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115bc7292b32b7442aadca724e68c55f3f_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:52,993 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/5957983afeb14b279bddd74c275e8c37, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:52,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/5957983afeb14b279bddd74c275e8c37 is 175, key is test_row_0/A:col10/1731307371840/Put/seqid=0 2024-11-11T06:42:52,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742378_1554 (size=39749) 2024-11-11T06:42:53,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307433012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307433015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307433017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307433119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307433129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307433129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307433329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307433339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307433339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,398 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/5957983afeb14b279bddd74c275e8c37 2024-11-11T06:42:53,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/97de5a073035477a857344420077d445 is 50, key is test_row_0/B:col10/1731307371840/Put/seqid=0 2024-11-11T06:42:53,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742379_1555 (size=12151) 2024-11-11T06:42:53,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307433639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307433649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307433649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:53,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/97de5a073035477a857344420077d445 2024-11-11T06:42:53,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/e8c16b1bacec4c34896ffec1f5b8fba5 is 50, key is test_row_0/C:col10/1731307371840/Put/seqid=0 2024-11-11T06:42:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742380_1556 (size=12151) 2024-11-11T06:42:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40198 deadline: 1731307434149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:54,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40244 deadline: 1731307434159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:54,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T06:42:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35807 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40228 deadline: 1731307434159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 , the old value is region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T06:42:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., hostname=ce2cf0244921,35807,1731307183669, seqNum=5 because the exception is null or not the one we care about 2024-11-11T06:42:54,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/e8c16b1bacec4c34896ffec1f5b8fba5 2024-11-11T06:42:54,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/5957983afeb14b279bddd74c275e8c37 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5957983afeb14b279bddd74c275e8c37 2024-11-11T06:42:54,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5957983afeb14b279bddd74c275e8c37, entries=200, sequenceid=212, filesize=38.8 K 2024-11-11T06:42:54,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/97de5a073035477a857344420077d445 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/97de5a073035477a857344420077d445 2024-11-11T06:42:54,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/97de5a073035477a857344420077d445, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T06:42:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/e8c16b1bacec4c34896ffec1f5b8fba5 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e8c16b1bacec4c34896ffec1f5b8fba5 2024-11-11T06:42:54,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e8c16b1bacec4c34896ffec1f5b8fba5, entries=150, sequenceid=212, filesize=11.9 K 2024-11-11T06:42:54,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 8d454a2f13681bc747ece68bc181872e in 1255ms, sequenceid=212, compaction requested=true 2024-11-11T06:42:54,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T06:42:54,237 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T06:42:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:54,237 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8d454a2f13681bc747ece68bc181872e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T06:42:54,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:54,237 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:54,237 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:54,237 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/B is initiating minor compaction (all files) 2024-11-11T06:42:54,237 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/A is initiating minor compaction (all files) 2024-11-11T06:42:54,237 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/B in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:54,237 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/A in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:54,237 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0af3d5a5fb60415ebd98a1b196eb2cad, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/277f7be4808c4b1082b74f0eb38906d9, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/97de5a073035477a857344420077d445] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=36.0 K 2024-11-11T06:42:54,238 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/49b04b00eec24f6ea5d861767c6f24b6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/a10ac059ae5c46b884730b9e498ac67c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5957983afeb14b279bddd74c275e8c37] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=99.9 K 2024-11-11T06:42:54,238 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:54,238 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af3d5a5fb60415ebd98a1b196eb2cad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307368504 2024-11-11T06:42:54,238 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. files: [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/49b04b00eec24f6ea5d861767c6f24b6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/a10ac059ae5c46b884730b9e498ac67c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5957983afeb14b279bddd74c275e8c37] 2024-11-11T06:42:54,238 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 277f7be4808c4b1082b74f0eb38906d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731307369661 2024-11-11T06:42:54,238 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49b04b00eec24f6ea5d861767c6f24b6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307368504 2024-11-11T06:42:54,238 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting 97de5a073035477a857344420077d445, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731307371840 2024-11-11T06:42:54,238 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting a10ac059ae5c46b884730b9e498ac67c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731307369661 2024-11-11T06:42:54,239 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5957983afeb14b279bddd74c275e8c37, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731307371839 2024-11-11T06:42:54,242 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:54,243 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:54,243 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/be246c7e029c493e8511435ab17d108b is 50, key is test_row_0/B:col10/1731307371840/Put/seqid=0 2024-11-11T06:42:54,244 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111adc706a6e4594a45af5b078ee5944572_8d454a2f13681bc747ece68bc181872e store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:54,245 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111adc706a6e4594a45af5b078ee5944572_8d454a2f13681bc747ece68bc181872e, store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:54,245 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111adc706a6e4594a45af5b078ee5944572_8d454a2f13681bc747ece68bc181872e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:54,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742381_1557 (size=12629) 2024-11-11T06:42:54,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742382_1558 (size=4469) 2024-11-11T06:42:54,252 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/be246c7e029c493e8511435ab17d108b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/be246c7e029c493e8511435ab17d108b 2024-11-11T06:42:54,256 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/B of 8d454a2f13681bc747ece68bc181872e into be246c7e029c493e8511435ab17d108b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:54,256 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:54,256 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/B, priority=13, startTime=1731307374237; duration=0sec 2024-11-11T06:42:54,256 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T06:42:54,256 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:B 2024-11-11T06:42:54,256 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T06:42:54,257 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T06:42:54,257 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1541): 8d454a2f13681bc747ece68bc181872e/C is initiating minor compaction (all files) 2024-11-11T06:42:54,257 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8d454a2f13681bc747ece68bc181872e/C in TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:54,257 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/fb077e550b7441b78ee4e2aebd1d6e86, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b3fc4052eb0c416088a96e05da6ff93c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e8c16b1bacec4c34896ffec1f5b8fba5] into tmpdir=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp, totalSize=36.0 K 2024-11-11T06:42:54,257 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting fb077e550b7441b78ee4e2aebd1d6e86, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731307368504 2024-11-11T06:42:54,257 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting b3fc4052eb0c416088a96e05da6ff93c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731307369661 2024-11-11T06:42:54,258 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] compactions.Compactor(225): Compacting e8c16b1bacec4c34896ffec1f5b8fba5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731307371840 2024-11-11T06:42:54,263 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:54,264 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/7d0a96941bdc4024a4a738562bdd940a is 50, key is test_row_0/C:col10/1731307371840/Put/seqid=0 2024-11-11T06:42:54,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742383_1559 (size=12629) 2024-11-11T06:42:54,275 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/7d0a96941bdc4024a4a738562bdd940a as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/7d0a96941bdc4024a4a738562bdd940a 2024-11-11T06:42:54,278 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/C of 8d454a2f13681bc747ece68bc181872e into 7d0a96941bdc4024a4a738562bdd940a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:54,278 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:54,278 INFO [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/C, priority=13, startTime=1731307374237; duration=0sec 2024-11-11T06:42:54,278 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:54,278 DEBUG [RS:0;ce2cf0244921:35807-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:C 2024-11-11T06:42:54,650 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8d454a2f13681bc747ece68bc181872e#A#compaction#466 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T06:42:54,651 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/f2a8c20674634abf92c7e57fd002de01 is 175, key is test_row_0/A:col10/1731307371840/Put/seqid=0 2024-11-11T06:42:54,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742384_1560 (size=31583) 2024-11-11T06:42:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T06:42:54,679 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:54,679 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T06:42:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-11T06:42:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T06:42:54,681 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T06:42:54,681 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T06:42:54,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T06:42:54,729 INFO [Thread-2271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2271. 2024-11-11T06:42:54,729 DEBUG [Thread-2271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:54,729 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,729 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:54,730 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-11-11T06:42:54,730 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:54,730 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,730 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:54,730 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-11-11T06:42:54,730 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:54,730 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,730 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:54,731 INFO [Thread-2265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2265. 2024-11-11T06:42:54,731 DEBUG [Thread-2265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:54,731 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,731 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:54,732 INFO [Thread-2269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2269. 2024-11-11T06:42:54,732 DEBUG [Thread-2269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:54,732 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,732 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:54,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T06:42:54,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35807 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T06:42:54,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:54,834 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-11T06:42:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117de653adeac6401b954986f8c39c5775_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307373010/Put/seqid=0 2024-11-11T06:42:54,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742385_1561 (size=12304) 2024-11-11T06:42:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T06:42:55,101 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/f2a8c20674634abf92c7e57fd002de01 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/f2a8c20674634abf92c7e57fd002de01 2024-11-11T06:42:55,109 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8d454a2f13681bc747ece68bc181872e/A of 8d454a2f13681bc747ece68bc181872e into f2a8c20674634abf92c7e57fd002de01(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T06:42:55,109 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:55,109 INFO [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., storeName=8d454a2f13681bc747ece68bc181872e/A, priority=13, startTime=1731307374237; duration=0sec 2024-11-11T06:42:55,109 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T06:42:55,109 DEBUG [RS:0;ce2cf0244921:35807-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8d454a2f13681bc747ece68bc181872e:A 2024-11-11T06:42:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35807 {}] regionserver.HRegion(8855): Flush requested on 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:55,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. as already flushing 2024-11-11T06:42:55,173 INFO [Thread-2258 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2258. 2024-11-11T06:42:55,173 DEBUG [Thread-2258 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:55,173 DEBUG [Thread-2258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:55,174 DEBUG [Thread-2258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:55,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:55,183 INFO [Thread-2254 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2254. 2024-11-11T06:42:55,183 INFO [Thread-2256 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2256. 2024-11-11T06:42:55,183 DEBUG [Thread-2254 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:55,183 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:55,183 DEBUG [Thread-2256 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:55,183 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:55,183 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:55,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:55,184 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:55,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:55,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:55,254 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117de653adeac6401b954986f8c39c5775_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117de653adeac6401b954986f8c39c5775_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:55,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/64287b1d4bab4fbb8b8ab51a2d4c8ce1, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:55,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/64287b1d4bab4fbb8b8ab51a2d4c8ce1 is 175, key is test_row_0/A:col10/1731307373010/Put/seqid=0 2024-11-11T06:42:55,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742386_1562 (size=31105) 2024-11-11T06:42:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T06:42:55,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/64287b1d4bab4fbb8b8ab51a2d4c8ce1 2024-11-11T06:42:55,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/5c9c50a2b8814d4bb40729df54f533cf is 50, key is test_row_0/B:col10/1731307373010/Put/seqid=0 2024-11-11T06:42:55,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742387_1563 (size=12151) 2024-11-11T06:42:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T06:42:56,083 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/5c9c50a2b8814d4bb40729df54f533cf 2024-11-11T06:42:56,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/e14c5b81447e4d19879e4bddacb82f86 is 50, key is test_row_0/C:col10/1731307373010/Put/seqid=0 2024-11-11T06:42:56,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742388_1564 (size=12151) 2024-11-11T06:42:56,500 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/e14c5b81447e4d19879e4bddacb82f86 2024-11-11T06:42:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/64287b1d4bab4fbb8b8ab51a2d4c8ce1 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/64287b1d4bab4fbb8b8ab51a2d4c8ce1 2024-11-11T06:42:56,517 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/64287b1d4bab4fbb8b8ab51a2d4c8ce1, entries=150, sequenceid=233, filesize=30.4 K 2024-11-11T06:42:56,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/5c9c50a2b8814d4bb40729df54f533cf as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/5c9c50a2b8814d4bb40729df54f533cf 2024-11-11T06:42:56,524 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/5c9c50a2b8814d4bb40729df54f533cf, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T06:42:56,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/e14c5b81447e4d19879e4bddacb82f86 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e14c5b81447e4d19879e4bddacb82f86 2024-11-11T06:42:56,530 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e14c5b81447e4d19879e4bddacb82f86, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T06:42:56,531 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=20.13 KB/20610 for 8d454a2f13681bc747ece68bc181872e in 1697ms, sequenceid=233, compaction requested=false 2024-11-11T06:42:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 8d454a2f13681bc747ece68bc181872e: 2024-11-11T06:42:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-11T06:42:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-11T06:42:56,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T06:42:56,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8510 sec 2024-11-11T06:42:56,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.8560 sec 2024-11-11T06:42:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T06:42:56,830 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:57,390 INFO [Thread-2262 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2262. 2024-11-11T06:42:57,390 DEBUG [Thread-2262 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:57,390 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:57,390 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:57,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:57,432 INFO [Thread-2260 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2260. 2024-11-11T06:42:57,432 DEBUG [Thread-2260 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T06:42:57,432 DEBUG [Thread-2260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:57,432 DEBUG [Thread-2260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:57,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:57,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T06:42:57,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-11T06:42:57,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-11T06:42:57,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-11T06:42:57,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-11T06:42:57,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-11T06:42:57,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T06:42:57,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4162 2024-11-11T06:42:57,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4075 2024-11-11T06:42:57,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4116 2024-11-11T06:42:57,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4167 2024-11-11T06:42:57,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4071 2024-11-11T06:42:57,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T06:42:57,433 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T06:42:57,433 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:42:57,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:57,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:57,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:57,434 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T06:42:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T06:42:57,439 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307377439"}]},"ts":"1731307377439"} 2024-11-11T06:42:57,441 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T06:42:57,441 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T06:42:57,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T06:42:57,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, UNASSIGN}] 2024-11-11T06:42:57,446 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, UNASSIGN 2024-11-11T06:42:57,447 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=8d454a2f13681bc747ece68bc181872e, regionState=CLOSING, regionLocation=ce2cf0244921,35807,1731307183669 2024-11-11T06:42:57,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, UNASSIGN because future has completed 2024-11-11T06:42:57,450 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T06:42:57,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669}] 2024-11-11T06:42:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T06:42:57,604 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:57,604 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T06:42:57,604 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 8d454a2f13681bc747ece68bc181872e, disabling compactions & flushes 2024-11-11T06:42:57,604 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:57,604 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:57,604 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. after waiting 0 ms 2024-11-11T06:42:57,604 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:57,605 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 8d454a2f13681bc747ece68bc181872e 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T06:42:57,612 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111270f869d1add44f6974edb70476aaed5_8d454a2f13681bc747ece68bc181872e is 50, key is test_row_0/A:col10/1731307377430/Put/seqid=0 2024-11-11T06:42:57,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742389_1565 (size=9814) 2024-11-11T06:42:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T06:42:58,016 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T06:42:58,026 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111270f869d1add44f6974edb70476aaed5_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111270f869d1add44f6974edb70476aaed5_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:58,027 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fbb995577b4d4ca0a11f52f1e026a825, store: [table=TestAcidGuarantees family=A region=8d454a2f13681bc747ece68bc181872e] 2024-11-11T06:42:58,028 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fbb995577b4d4ca0a11f52f1e026a825 is 175, key is test_row_0/A:col10/1731307377430/Put/seqid=0 2024-11-11T06:42:58,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742390_1566 (size=22461) 2024-11-11T06:42:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T06:42:58,433 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fbb995577b4d4ca0a11f52f1e026a825 2024-11-11T06:42:58,443 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/d557679bf65c4137bac7caf7ab087c1b is 50, key is test_row_0/B:col10/1731307377430/Put/seqid=0 2024-11-11T06:42:58,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742391_1567 (size=9757) 2024-11-11T06:42:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T06:42:58,848 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/d557679bf65c4137bac7caf7ab087c1b 2024-11-11T06:42:58,861 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/58097687d96546d2a85aed790e7ac510 is 50, key is test_row_0/C:col10/1731307377430/Put/seqid=0 2024-11-11T06:42:58,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742392_1568 (size=9757) 2024-11-11T06:42:59,266 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/58097687d96546d2a85aed790e7ac510 2024-11-11T06:42:59,274 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/A/fbb995577b4d4ca0a11f52f1e026a825 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fbb995577b4d4ca0a11f52f1e026a825 2024-11-11T06:42:59,279 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fbb995577b4d4ca0a11f52f1e026a825, entries=100, sequenceid=242, filesize=21.9 K 2024-11-11T06:42:59,280 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/B/d557679bf65c4137bac7caf7ab087c1b as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/d557679bf65c4137bac7caf7ab087c1b 2024-11-11T06:42:59,283 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/d557679bf65c4137bac7caf7ab087c1b, entries=100, sequenceid=242, filesize=9.5 K 2024-11-11T06:42:59,284 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/.tmp/C/58097687d96546d2a85aed790e7ac510 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/58097687d96546d2a85aed790e7ac510 2024-11-11T06:42:59,286 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/58097687d96546d2a85aed790e7ac510, entries=100, sequenceid=242, filesize=9.5 K 2024-11-11T06:42:59,287 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8d454a2f13681bc747ece68bc181872e in 1683ms, sequenceid=242, compaction requested=true 2024-11-11T06:42:59,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/e171a62c62f3409289f50de13232dbb1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/df04dffdf7814f1f87b8bbe30fd4710b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5eb663be0488476ab360a70446f6ce6b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/1e443970822a46258f7462da9832ff52, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fd4b689301f54bcd81bd85d207bc63f7, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/b2d2898b5f024620a544b3a409ffbaca, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/9d8d22a2bf42471a9a0c7b0436e6d1fe, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/06a6599886a644779ebf760877521c14, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/2ce97949350f4100a50b82a325cc8cda, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/d47982b3d1334685881f06c1015cd747, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fc65dcf89514492dbd093abbcab28287, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/49b04b00eec24f6ea5d861767c6f24b6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/a10ac059ae5c46b884730b9e498ac67c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5957983afeb14b279bddd74c275e8c37] to archive 2024-11-11T06:42:59,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:59,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/e171a62c62f3409289f50de13232dbb1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/e171a62c62f3409289f50de13232dbb1 2024-11-11T06:42:59,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/df04dffdf7814f1f87b8bbe30fd4710b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/df04dffdf7814f1f87b8bbe30fd4710b 2024-11-11T06:42:59,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5eb663be0488476ab360a70446f6ce6b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5eb663be0488476ab360a70446f6ce6b 2024-11-11T06:42:59,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/1e443970822a46258f7462da9832ff52 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/1e443970822a46258f7462da9832ff52 2024-11-11T06:42:59,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fd4b689301f54bcd81bd85d207bc63f7 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fd4b689301f54bcd81bd85d207bc63f7 2024-11-11T06:42:59,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/b2d2898b5f024620a544b3a409ffbaca to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/b2d2898b5f024620a544b3a409ffbaca 2024-11-11T06:42:59,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/9d8d22a2bf42471a9a0c7b0436e6d1fe to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/9d8d22a2bf42471a9a0c7b0436e6d1fe 2024-11-11T06:42:59,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/06a6599886a644779ebf760877521c14 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/06a6599886a644779ebf760877521c14 2024-11-11T06:42:59,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/2ce97949350f4100a50b82a325cc8cda to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/2ce97949350f4100a50b82a325cc8cda 2024-11-11T06:42:59,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/d47982b3d1334685881f06c1015cd747 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/d47982b3d1334685881f06c1015cd747 2024-11-11T06:42:59,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fc65dcf89514492dbd093abbcab28287 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fc65dcf89514492dbd093abbcab28287 2024-11-11T06:42:59,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/49b04b00eec24f6ea5d861767c6f24b6 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/49b04b00eec24f6ea5d861767c6f24b6 2024-11-11T06:42:59,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/a10ac059ae5c46b884730b9e498ac67c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/a10ac059ae5c46b884730b9e498ac67c 2024-11-11T06:42:59,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5957983afeb14b279bddd74c275e8c37 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/5957983afeb14b279bddd74c275e8c37 2024-11-11T06:42:59,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/c0a73f3cfdd6448baac10da226bf23bd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/42b438eccfa849779273242c869349cd, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0cfcc86f725f43e2a4ff9cb919633a7d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/9ad23339090443f8b07a211dbdee95d8, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/eb099bcdab61403c91ca736ecc460aa1, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/3694222e3eae45a4a2902b5eafefefb0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/ba9bce30660e4ca199b465f535503d8e, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/e079913a84154dc589fa34eef893d134, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/36239d72cd6d4f559d495a54d1664492, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/a0ae0761fbfc4cd5b905775e979d274f, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0af3d5a5fb60415ebd98a1b196eb2cad, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/bdcecc07b1564d01b1f3e81678e2aab0, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/277f7be4808c4b1082b74f0eb38906d9, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/97de5a073035477a857344420077d445] to archive 2024-11-11T06:42:59,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:59,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/c0a73f3cfdd6448baac10da226bf23bd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/c0a73f3cfdd6448baac10da226bf23bd 2024-11-11T06:42:59,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/42b438eccfa849779273242c869349cd to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/42b438eccfa849779273242c869349cd 2024-11-11T06:42:59,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0cfcc86f725f43e2a4ff9cb919633a7d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0cfcc86f725f43e2a4ff9cb919633a7d 2024-11-11T06:42:59,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/9ad23339090443f8b07a211dbdee95d8 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/9ad23339090443f8b07a211dbdee95d8 2024-11-11T06:42:59,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/eb099bcdab61403c91ca736ecc460aa1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/eb099bcdab61403c91ca736ecc460aa1 2024-11-11T06:42:59,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/3694222e3eae45a4a2902b5eafefefb0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/3694222e3eae45a4a2902b5eafefefb0 2024-11-11T06:42:59,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/ba9bce30660e4ca199b465f535503d8e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/ba9bce30660e4ca199b465f535503d8e 2024-11-11T06:42:59,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/e079913a84154dc589fa34eef893d134 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/e079913a84154dc589fa34eef893d134 2024-11-11T06:42:59,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/36239d72cd6d4f559d495a54d1664492 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/36239d72cd6d4f559d495a54d1664492 2024-11-11T06:42:59,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/a0ae0761fbfc4cd5b905775e979d274f to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/a0ae0761fbfc4cd5b905775e979d274f 2024-11-11T06:42:59,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0af3d5a5fb60415ebd98a1b196eb2cad to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/0af3d5a5fb60415ebd98a1b196eb2cad 2024-11-11T06:42:59,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/bdcecc07b1564d01b1f3e81678e2aab0 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/bdcecc07b1564d01b1f3e81678e2aab0 2024-11-11T06:42:59,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/277f7be4808c4b1082b74f0eb38906d9 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/277f7be4808c4b1082b74f0eb38906d9 2024-11-11T06:42:59,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/97de5a073035477a857344420077d445 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/97de5a073035477a857344420077d445 2024-11-11T06:42:59,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/df09468e8f5d4dfc9e55690113f995f9, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/39516f2df5094487a21cc482f4dda9d9, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/f6393a7e350f40beac70363b49f346c3, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8e7c057b497d4540b5ac52982d8c5170, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/97cf153313d14df0838118a43b98d608, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8c1379ab816540d7a2aadfedc0c07b5b, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b2b6df3f14a542169ab0f2b8ae31909c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/76dfe4bf70264fd08f9888d8878fb66d, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b03eb1cf072b4bd5b50c04a40ff847c5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/5fc59227991f4b34aa70995b8f7317d5, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/fb077e550b7441b78ee4e2aebd1d6e86, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/4122a4f508584ec0a6868a3fc9ab18b6, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b3fc4052eb0c416088a96e05da6ff93c, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e8c16b1bacec4c34896ffec1f5b8fba5] to archive 2024-11-11T06:42:59,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T06:42:59,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/df09468e8f5d4dfc9e55690113f995f9 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/df09468e8f5d4dfc9e55690113f995f9 2024-11-11T06:42:59,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/39516f2df5094487a21cc482f4dda9d9 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/39516f2df5094487a21cc482f4dda9d9 2024-11-11T06:42:59,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/f6393a7e350f40beac70363b49f346c3 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/f6393a7e350f40beac70363b49f346c3 2024-11-11T06:42:59,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8e7c057b497d4540b5ac52982d8c5170 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8e7c057b497d4540b5ac52982d8c5170 2024-11-11T06:42:59,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/97cf153313d14df0838118a43b98d608 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/97cf153313d14df0838118a43b98d608 2024-11-11T06:42:59,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8c1379ab816540d7a2aadfedc0c07b5b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/8c1379ab816540d7a2aadfedc0c07b5b 2024-11-11T06:42:59,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b2b6df3f14a542169ab0f2b8ae31909c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b2b6df3f14a542169ab0f2b8ae31909c 2024-11-11T06:42:59,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/76dfe4bf70264fd08f9888d8878fb66d to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/76dfe4bf70264fd08f9888d8878fb66d 2024-11-11T06:42:59,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b03eb1cf072b4bd5b50c04a40ff847c5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b03eb1cf072b4bd5b50c04a40ff847c5 2024-11-11T06:42:59,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/5fc59227991f4b34aa70995b8f7317d5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/5fc59227991f4b34aa70995b8f7317d5 2024-11-11T06:42:59,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/fb077e550b7441b78ee4e2aebd1d6e86 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/fb077e550b7441b78ee4e2aebd1d6e86 2024-11-11T06:42:59,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/4122a4f508584ec0a6868a3fc9ab18b6 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/4122a4f508584ec0a6868a3fc9ab18b6 2024-11-11T06:42:59,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b3fc4052eb0c416088a96e05da6ff93c to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/b3fc4052eb0c416088a96e05da6ff93c 2024-11-11T06:42:59,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e8c16b1bacec4c34896ffec1f5b8fba5 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e8c16b1bacec4c34896ffec1f5b8fba5 2024-11-11T06:42:59,325 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/recovered.edits/245.seqid, newMaxSeqId=245, maxSeqId=4 2024-11-11T06:42:59,326 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. 2024-11-11T06:42:59,326 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 8d454a2f13681bc747ece68bc181872e: Waiting for close lock at 1731307377604Running coprocessor pre-close hooks at 1731307377604Disabling compacts and flushes for region at 1731307377604Disabling writes for close at 1731307377604Obtaining lock to block concurrent updates at 1731307377605 (+1 ms)Preparing flush snapshotting stores in 8d454a2f13681bc747ece68bc181872e at 1731307377605Finished memstore snapshotting TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731307377605Flushing stores of TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e. at 1731307377606 (+1 ms)Flushing 8d454a2f13681bc747ece68bc181872e/A: creating writer at 1731307377606Flushing mob file 8d454a2f13681bc747ece68bc181872e/A: appending metadata at 1731307377612 (+6 ms)Flushing mob file 8d454a2f13681bc747ece68bc181872e/A: closing flushed file at 1731307377612Flushing 8d454a2f13681bc747ece68bc181872e/A: appending metadata at 1731307378027 (+415 ms)Flushing 8d454a2f13681bc747ece68bc181872e/A: closing flushed file at 1731307378027Flushing 8d454a2f13681bc747ece68bc181872e/B: creating writer at 1731307378440 (+413 ms)Flushing 8d454a2f13681bc747ece68bc181872e/B: appending metadata at 1731307378443 (+3 ms)Flushing 8d454a2f13681bc747ece68bc181872e/B: closing flushed file at 1731307378443Flushing 8d454a2f13681bc747ece68bc181872e/C: creating writer at 1731307378857 (+414 ms)Flushing 8d454a2f13681bc747ece68bc181872e/C: appending metadata at 1731307378861 (+4 ms)Flushing 8d454a2f13681bc747ece68bc181872e/C: closing flushed file at 1731307378861Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@405b406: reopening flushed file at 1731307379273 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6443b602: reopening flushed file at 1731307379279 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a276d9b: reopening flushed file at 1731307379283 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8d454a2f13681bc747ece68bc181872e in 1683ms, sequenceid=242, compaction requested=true at 1731307379287 (+4 ms)Writing region close event to WAL at 1731307379323 (+36 ms)Running coprocessor post-close hooks at 1731307379325 (+2 ms)Closed at 1731307379325 2024-11-11T06:42:59,327 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,327 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=8d454a2f13681bc747ece68bc181872e, regionState=CLOSED 2024-11-11T06:42:59,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 because future has completed 2024-11-11T06:42:59,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-11T06:42:59,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 8d454a2f13681bc747ece68bc181872e, server=ce2cf0244921,35807,1731307183669 in 1.8790 sec 2024-11-11T06:42:59,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-11T06:42:59,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8d454a2f13681bc747ece68bc181872e, UNASSIGN in 1.8850 sec 2024-11-11T06:42:59,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-11T06:42:59,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8890 sec 2024-11-11T06:42:59,333 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731307379333"}]},"ts":"1731307379333"} 2024-11-11T06:42:59,334 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T06:42:59,334 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T06:42:59,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9000 sec 2024-11-11T06:42:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T06:42:59,600 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:59,600 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T06:42:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:59,602 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T06:42:59,603 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:59,607 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,610 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C, FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/recovered.edits] 2024-11-11T06:42:59,614 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/64287b1d4bab4fbb8b8ab51a2d4c8ce1 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/64287b1d4bab4fbb8b8ab51a2d4c8ce1 2024-11-11T06:42:59,616 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/f2a8c20674634abf92c7e57fd002de01 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/f2a8c20674634abf92c7e57fd002de01 2024-11-11T06:42:59,619 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fbb995577b4d4ca0a11f52f1e026a825 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/A/fbb995577b4d4ca0a11f52f1e026a825 2024-11-11T06:42:59,623 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/5c9c50a2b8814d4bb40729df54f533cf to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/5c9c50a2b8814d4bb40729df54f533cf 2024-11-11T06:42:59,625 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/be246c7e029c493e8511435ab17d108b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/be246c7e029c493e8511435ab17d108b 2024-11-11T06:42:59,626 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/d557679bf65c4137bac7caf7ab087c1b to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/B/d557679bf65c4137bac7caf7ab087c1b 2024-11-11T06:42:59,629 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/58097687d96546d2a85aed790e7ac510 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/58097687d96546d2a85aed790e7ac510 2024-11-11T06:42:59,630 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/7d0a96941bdc4024a4a738562bdd940a to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/7d0a96941bdc4024a4a738562bdd940a 2024-11-11T06:42:59,632 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e14c5b81447e4d19879e4bddacb82f86 to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/C/e14c5b81447e4d19879e4bddacb82f86 2024-11-11T06:42:59,634 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/recovered.edits/245.seqid to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e/recovered.edits/245.seqid 2024-11-11T06:42:59,635 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/default/TestAcidGuarantees/8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,635 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T06:42:59,635 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T06:42:59,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T06:42:59,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111855cbeea12a4f0bbb8acc1105fc325a_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111855cbeea12a4f0bbb8acc1105fc325a_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111270f869d1add44f6974edb70476aaed5_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111270f869d1add44f6974edb70476aaed5_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112752eaed3fb14c7cb4251726546d90a1_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112752eaed3fb14c7cb4251726546d90a1_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149c8b8b2fe8b40399560b99dd1c3e2fb_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149c8b8b2fe8b40399560b99dd1c3e2fb_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115bc7292b32b7442aadca724e68c55f3f_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115bc7292b32b7442aadca724e68c55f3f_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117127da53b5f141ff905291b8c2bae8b7_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117127da53b5f141ff905291b8c2bae8b7_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111171fe9ec1e13447b38a555f80e6c43659_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111171fe9ec1e13447b38a555f80e6c43659_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117de653adeac6401b954986f8c39c5775_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117de653adeac6401b954986f8c39c5775_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ed0a11d901a4b638b301e5f10292bf4_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117ed0a11d901a4b638b301e5f10292bf4_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111181d52800647b44498652ce791aad98ae_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111181d52800647b44498652ce791aad98ae_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111955b6c307a184b5f9da2d3f305697e90_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111955b6c307a184b5f9da2d3f305697e90_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,648 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aacf13bf982d46e382eeb53884568583_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aacf13bf982d46e382eeb53884568583_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c4e490d953e84671ab0b8438d8890e07_8d454a2f13681bc747ece68bc181872e to hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c4e490d953e84671ab0b8438d8890e07_8d454a2f13681bc747ece68bc181872e 2024-11-11T06:42:59,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T06:42:59,653 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:59,656 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T06:42:59,658 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T06:42:59,659 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:59,659 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T06:42:59,659 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731307379659"}]},"ts":"9223372036854775807"} 2024-11-11T06:42:59,661 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T06:42:59,661 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8d454a2f13681bc747ece68bc181872e, NAME => 'TestAcidGuarantees,,1731307351577.8d454a2f13681bc747ece68bc181872e.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T06:42:59,661 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T06:42:59,661 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731307379661"}]},"ts":"9223372036854775807"} 2024-11-11T06:42:59,662 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T06:42:59,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T06:42:59,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-11T06:42:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40845 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T06:42:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T06:42:59,710 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T06:42:59,719 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 241), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 353), ProcessCount=11 (was 11), AvailableMemoryMB=10845 (was 10851) 2024-11-11T06:42:59,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T06:42:59,719 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T06:42:59,719 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:42:59,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:59,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:59,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T06:42:59,720 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T06:42:59,720 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=164229068, stopped=false 2024-11-11T06:42:59,720 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ce2cf0244921,40845,1731307182847 2024-11-11T06:42:59,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T06:42:59,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T06:42:59,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:42:59,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:42:59,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T06:42:59,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T06:42:59,775 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:42:59,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:59,776 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T06:42:59,776 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ce2cf0244921,35807,1731307183669' ***** 2024-11-11T06:42:59,776 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T06:42:59,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T06:42:59,777 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T06:42:59,777 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T06:42:59,778 INFO [RS:0;ce2cf0244921:35807 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T06:42:59,778 INFO [RS:0;ce2cf0244921:35807 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T06:42:59,778 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(959): stopping server ce2cf0244921,35807,1731307183669 2024-11-11T06:42:59,778 INFO [RS:0;ce2cf0244921:35807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T06:42:59,778 INFO [RS:0;ce2cf0244921:35807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ce2cf0244921:35807. 2024-11-11T06:42:59,778 DEBUG [RS:0;ce2cf0244921:35807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T06:42:59,778 DEBUG [RS:0;ce2cf0244921:35807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:42:59,779 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T06:42:59,779 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T06:42:59,779 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T06:42:59,779 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T06:42:59,780 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T06:42:59,780 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T06:42:59,780 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T06:42:59,780 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:42:59,780 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T06:42:59,780 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T06:42:59,780 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T06:42:59,780 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T06:42:59,780 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T06:42:59,800 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/info/5d1084fcb5014cde93cb83870979118d is 95, key is TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7./info:/1731307222205/DeleteFamily/seqid=0 2024-11-11T06:42:59,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742393_1569 (size=6514) 2024-11-11T06:42:59,817 INFO [regionserver/ce2cf0244921:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T06:42:59,980 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:42:59,985 INFO [regionserver/ce2cf0244921:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T06:42:59,985 INFO [regionserver/ce2cf0244921:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T06:43:00,181 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:43:00,204 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/info/5d1084fcb5014cde93cb83870979118d 2024-11-11T06:43:00,252 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/ns/08239e0718234901a6933f969557c195 is 93, key is TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7./ns:/1731307222169/DeleteFamily/seqid=0 2024-11-11T06:43:00,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742394_1570 (size=5906) 2024-11-11T06:43:00,381 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:43:00,559 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ce2cf0244921:40845 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-11T06:43:00,581 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:43:00,656 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/ns/08239e0718234901a6933f969557c195 2024-11-11T06:43:00,682 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/rep_barrier/e354a86c98474bad9b3508dd0184c2ca is 102, key is TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7./rep_barrier:/1731307222169/DeleteFamily/seqid=0 2024-11-11T06:43:00,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742395_1571 (size=6025) 2024-11-11T06:43:00,781 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:43:00,982 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:43:01,086 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/rep_barrier/e354a86c98474bad9b3508dd0184c2ca 2024-11-11T06:43:01,112 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/table/c33be48fb80849069d8d259a4c0253f4 is 96, key is TestAcidGuarantees,,1731307187914.1b6d451b960d07000ebc207f625528a7./table:/1731307222169/DeleteFamily/seqid=0 2024-11-11T06:43:01,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742396_1572 (size=5936) 2024-11-11T06:43:01,182 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:43:01,382 DEBUG [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T06:43:01,516 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/table/c33be48fb80849069d8d259a4c0253f4 2024-11-11T06:43:01,527 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/info/5d1084fcb5014cde93cb83870979118d as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/info/5d1084fcb5014cde93cb83870979118d 2024-11-11T06:43:01,533 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/info/5d1084fcb5014cde93cb83870979118d, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T06:43:01,534 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/ns/08239e0718234901a6933f969557c195 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/ns/08239e0718234901a6933f969557c195 2024-11-11T06:43:01,538 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/ns/08239e0718234901a6933f969557c195, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T06:43:01,538 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/rep_barrier/e354a86c98474bad9b3508dd0184c2ca as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/rep_barrier/e354a86c98474bad9b3508dd0184c2ca 2024-11-11T06:43:01,542 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/rep_barrier/e354a86c98474bad9b3508dd0184c2ca, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T06:43:01,542 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/.tmp/table/c33be48fb80849069d8d259a4c0253f4 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/table/c33be48fb80849069d8d259a4c0253f4 2024-11-11T06:43:01,545 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/table/c33be48fb80849069d8d259a4c0253f4, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T06:43:01,546 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1765ms, sequenceid=90, compaction requested=false 2024-11-11T06:43:01,549 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T06:43:01,549 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T06:43:01,550 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T06:43:01,550 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731307379780Running coprocessor pre-close hooks at 1731307379780Disabling compacts and flushes for region at 1731307379780Disabling writes for close at 1731307379780Obtaining lock to block concurrent updates at 1731307379780Preparing flush snapshotting stores in 1588230740 at 1731307379780Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731307379781 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731307379781Flushing 1588230740/info: creating writer at 1731307379782 (+1 ms)Flushing 1588230740/info: appending metadata at 1731307379800 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731307379800Flushing 1588230740/ns: creating writer at 1731307380234 (+434 ms)Flushing 1588230740/ns: appending metadata at 1731307380252 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731307380252Flushing 1588230740/rep_barrier: creating writer at 1731307380663 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731307380681 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731307380681Flushing 1588230740/table: creating writer at 1731307381096 (+415 ms)Flushing 1588230740/table: appending metadata at 1731307381111 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731307381111Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d74417d: reopening flushed file at 1731307381525 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f7c91d8: reopening flushed file at 1731307381533 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eedb976: reopening flushed file at 1731307381538 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66bf93b: reopening flushed file at 1731307381542 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1765ms, sequenceid=90, compaction requested=false at 1731307381546 (+4 ms)Writing region close event to WAL at 1731307381547 (+1 ms)Running coprocessor post-close hooks at 1731307381549 (+2 ms)Closed at 1731307381550 (+1 ms) 2024-11-11T06:43:01,550 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T06:43:01,582 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(976): stopping server ce2cf0244921,35807,1731307183669; all regions closed. 2024-11-11T06:43:01,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741834_1010 (size=25344) 2024-11-11T06:43:01,589 DEBUG [RS:0;ce2cf0244921:35807 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/oldWALs 2024-11-11T06:43:01,589 INFO [RS:0;ce2cf0244921:35807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ce2cf0244921%2C35807%2C1731307183669.meta:.meta(num 1731307187424) 2024-11-11T06:43:01,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741833_1009 (size=12920487) 2024-11-11T06:43:01,593 DEBUG [RS:0;ce2cf0244921:35807 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/oldWALs 2024-11-11T06:43:01,593 INFO [RS:0;ce2cf0244921:35807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ce2cf0244921%2C35807%2C1731307183669:(num 1731307187041) 2024-11-11T06:43:01,593 DEBUG [RS:0;ce2cf0244921:35807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T06:43:01,593 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T06:43:01,593 INFO [RS:0;ce2cf0244921:35807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T06:43:01,593 INFO [RS:0;ce2cf0244921:35807 {}] hbase.ChoreService(370): Chore service for: regionserver/ce2cf0244921:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T06:43:01,593 INFO [RS:0;ce2cf0244921:35807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T06:43:01,593 INFO [regionserver/ce2cf0244921:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T06:43:01,594 INFO [RS:0;ce2cf0244921:35807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35807 2024-11-11T06:43:01,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T06:43:01,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ce2cf0244921,35807,1731307183669 2024-11-11T06:43:01,648 INFO [RS:0;ce2cf0244921:35807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T06:43:01,649 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ce2cf0244921,35807,1731307183669] 2024-11-11T06:43:01,664 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ce2cf0244921,35807,1731307183669 already deleted, retry=false 2024-11-11T06:43:01,664 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ce2cf0244921,35807,1731307183669 expired; onlineServers=0 2024-11-11T06:43:01,665 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ce2cf0244921,40845,1731307182847' ***** 2024-11-11T06:43:01,665 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T06:43:01,665 INFO [M:0;ce2cf0244921:40845 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T06:43:01,665 INFO [M:0;ce2cf0244921:40845 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T06:43:01,665 DEBUG [M:0;ce2cf0244921:40845 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T06:43:01,665 DEBUG [M:0;ce2cf0244921:40845 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T06:43:01,665 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T06:43:01,665 DEBUG [master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.small.0-1731307186655 {}] cleaner.HFileCleaner(306): Exit Thread[master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.small.0-1731307186655,5,FailOnTimeoutGroup] 2024-11-11T06:43:01,665 DEBUG [master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.large.0-1731307186653 {}] cleaner.HFileCleaner(306): Exit Thread[master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.large.0-1731307186653,5,FailOnTimeoutGroup] 2024-11-11T06:43:01,665 INFO [M:0;ce2cf0244921:40845 {}] hbase.ChoreService(370): Chore service for: master/ce2cf0244921:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T06:43:01,665 INFO [M:0;ce2cf0244921:40845 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T06:43:01,666 DEBUG [M:0;ce2cf0244921:40845 {}] master.HMaster(1795): Stopping service threads 2024-11-11T06:43:01,666 INFO [M:0;ce2cf0244921:40845 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T06:43:01,666 INFO [M:0;ce2cf0244921:40845 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T06:43:01,666 ERROR [M:0;ce2cf0244921:40845 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36813,5,PEWorkerGroup] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:36813 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36813,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T06:43:01,666 INFO [M:0;ce2cf0244921:40845 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T06:43:01,666 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T06:43:01,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T06:43:01,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T06:43:01,673 DEBUG [M:0;ce2cf0244921:40845 {}] zookeeper.ZKUtil(347): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T06:43:01,673 WARN [M:0;ce2cf0244921:40845 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T06:43:01,674 INFO [M:0;ce2cf0244921:40845 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/.lastflushedseqids 2024-11-11T06:43:01,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742397_1573 (size=72) 2024-11-11T06:43:01,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T06:43:01,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35807-0x10129e8aa610001, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T06:43:01,757 INFO [RS:0;ce2cf0244921:35807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T06:43:01,757 INFO [RS:0;ce2cf0244921:35807 {}] regionserver.HRegionServer(1031): Exiting; stopping=ce2cf0244921,35807,1731307183669; zookeeper connection closed. 2024-11-11T06:43:01,757 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3c281e51 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3c281e51 2024-11-11T06:43:01,758 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T06:43:02,082 INFO [M:0;ce2cf0244921:40845 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T06:43:02,083 INFO [M:0;ce2cf0244921:40845 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T06:43:02,083 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T06:43:02,083 INFO [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T06:43:02,083 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T06:43:02,083 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T06:43:02,083 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T06:43:02,083 INFO [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=730.38 KB heapSize=895.90 KB 2024-11-11T06:43:02,104 DEBUG [M:0;ce2cf0244921:40845 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5aae7966850141d4acec97a1e4ac97e2 is 82, key is hbase:meta,,1/info:regioninfo/1731307187500/Put/seqid=0 2024-11-11T06:43:02,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742398_1574 (size=5672) 2024-11-11T06:43:02,509 INFO [M:0;ce2cf0244921:40845 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2038 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5aae7966850141d4acec97a1e4ac97e2 2024-11-11T06:43:02,540 DEBUG [M:0;ce2cf0244921:40845 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8cc60f3d2d6341f892452b236e0ff6aa is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1731307354692/Put/seqid=0 2024-11-11T06:43:02,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742399_1575 (size=43152) 2024-11-11T06:43:02,945 INFO [M:0;ce2cf0244921:40845 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=729.82 KB at sequenceid=2038 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8cc60f3d2d6341f892452b236e0ff6aa 2024-11-11T06:43:02,954 INFO [M:0;ce2cf0244921:40845 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8cc60f3d2d6341f892452b236e0ff6aa 2024-11-11T06:43:02,970 DEBUG [M:0;ce2cf0244921:40845 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/515e01c62ad64f57ba56825a8b243db0 is 69, key is ce2cf0244921,35807,1731307183669/rs:state/1731307186795/Put/seqid=0 2024-11-11T06:43:02,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073742400_1576 (size=5156) 2024-11-11T06:43:03,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T06:43:03,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T06:43:03,262 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T06:43:03,374 INFO [M:0;ce2cf0244921:40845 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2038 (bloomFilter=true), to=hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/515e01c62ad64f57ba56825a8b243db0 2024-11-11T06:43:03,380 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5aae7966850141d4acec97a1e4ac97e2 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5aae7966850141d4acec97a1e4ac97e2 2024-11-11T06:43:03,386 INFO [M:0;ce2cf0244921:40845 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5aae7966850141d4acec97a1e4ac97e2, entries=8, sequenceid=2038, filesize=5.5 K 2024-11-11T06:43:03,386 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8cc60f3d2d6341f892452b236e0ff6aa as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8cc60f3d2d6341f892452b236e0ff6aa 2024-11-11T06:43:03,389 INFO [M:0;ce2cf0244921:40845 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8cc60f3d2d6341f892452b236e0ff6aa 2024-11-11T06:43:03,389 INFO [M:0;ce2cf0244921:40845 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8cc60f3d2d6341f892452b236e0ff6aa, entries=164, sequenceid=2038, filesize=42.1 K 2024-11-11T06:43:03,390 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/515e01c62ad64f57ba56825a8b243db0 as hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/515e01c62ad64f57ba56825a8b243db0 2024-11-11T06:43:03,392 INFO [M:0;ce2cf0244921:40845 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36813/user/jenkins/test-data/19d05264-ff3f-1186-f203-ee0c172dc416/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/515e01c62ad64f57ba56825a8b243db0, entries=1, sequenceid=2038, filesize=5.0 K 2024-11-11T06:43:03,393 INFO [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(3140): Finished flush of dataSize ~730.38 KB/747904, heapSize ~895.60 KB/917096, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1310ms, sequenceid=2038, compaction requested=false 2024-11-11T06:43:03,394 INFO [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T06:43:03,394 DEBUG [M:0;ce2cf0244921:40845 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731307382083Disabling compacts and flushes for region at 1731307382083Disabling writes for close at 1731307382083Obtaining lock to block concurrent updates at 1731307382083Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731307382083Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=747904, getHeapSize=917336, getOffHeapSize=0, getCellsCount=2227 at 1731307382084 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731307382085 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731307382085Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731307382104 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731307382104Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731307382519 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731307382539 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731307382539Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731307382954 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731307382969 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731307382969Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d6b469: reopening flushed file at 1731307383379 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ff230c5: reopening flushed file at 1731307383386 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eca43e2: reopening flushed file at 1731307383389 (+3 ms)Finished flush of dataSize ~730.38 KB/747904, heapSize ~895.60 KB/917096, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1310ms, sequenceid=2038, compaction requested=false at 1731307383393 (+4 ms)Writing region close event to WAL at 1731307383394 (+1 ms)Closed at 1731307383394 2024-11-11T06:43:03,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36917 is added to blk_1073741830_1006 (size=881549) 2024-11-11T06:43:03,396 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T06:43:03,396 INFO [M:0;ce2cf0244921:40845 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T06:43:03,397 INFO [M:0;ce2cf0244921:40845 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40845 2024-11-11T06:43:03,397 INFO [M:0;ce2cf0244921:40845 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T06:43:03,548 INFO [M:0;ce2cf0244921:40845 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T06:43:03,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T06:43:03,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40845-0x10129e8aa610000, quorum=127.0.0.1:64841, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T06:43:03,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T06:43:03,558 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T06:43:03,559 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T06:43:03,559 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T06:43:03,559 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/hadoop.log.dir/,STOPPED} 2024-11-11T06:43:03,562 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T06:43:03,562 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T06:43:03,563 WARN [BP-405970163-172.17.0.2-1731307179225 heartbeating to localhost/127.0.0.1:36813 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T06:43:03,563 WARN [BP-405970163-172.17.0.2-1731307179225 heartbeating to localhost/127.0.0.1:36813 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-405970163-172.17.0.2-1731307179225 (Datanode Uuid cc6721fe-723b-4614-bc5f-d27caf25327e) service to localhost/127.0.0.1:36813 2024-11-11T06:43:03,565 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/cluster_4468b6fc-5992-71c3-8666-e28d197c8ed7/data/data1/current/BP-405970163-172.17.0.2-1731307179225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T06:43:03,565 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/cluster_4468b6fc-5992-71c3-8666-e28d197c8ed7/data/data2/current/BP-405970163-172.17.0.2-1731307179225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T06:43:03,566 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T06:43:03,573 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T06:43:03,574 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T06:43:03,574 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T06:43:03,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T06:43:03,574 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ba0c382-ea69-3090-668b-12956c7ffd00/hadoop.log.dir/,STOPPED} 2024-11-11T06:43:03,588 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T06:43:03,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down